apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/md5.h>
  26 #include <apt-pkg/sha1.h>
  27 #include <apt-pkg/tagfile.h>
  28 #include <apt-pkg/indexrecords.h>
  29 #include <apt-pkg/metaindex.h>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <string>
  35 #include <sstream>
  36 #include <stdio.h>
  37 #include <ctime>
  38
  39 #include <apti18n.h>
  40                                                                         /*}}}*/
  41
  42 using namespace std;
  43
  44 // Acquire::Item::Item - Constructor                                    /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* */
  47 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  48                        PartialSize(0), Mode(0), ID(0), Complete(false),
  49                        Local(false), QueueCounter(0)
  50 {
  51    Owner->Add(this);
  52    Status = StatIdle;
  53 }
  54                                                                         /*}}}*/
  55 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 pkgAcquire::Item::~Item()
  59 {
  60    Owner->Remove(this);
  61 }
  62                                                                         /*}}}*/
  63 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* We return to an idle state if there are still other queues that could
  66    fetch this object */
  67 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  68 {
  69    Status = StatIdle;
  70    ErrorText = LookupTag(Message,"Message");
  71    UsedMirror =  LookupTag(Message,"UsedMirror");
  72    if (QueueCounter <= 1)
  73    {
  74       /* This indicates that the file is not available right now but might
  75          be sometime later. If we do a retry cycle then this should be
  76          retried [CDROMs] */
  77       if (Cnf->LocalOnly == true &&
  78           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  79       {
  80          Status = StatIdle;
  81          Dequeue();
  82          return;
  83       }
  84
  85       Status = StatError;
  86       Dequeue();
  87    }
  88
  89    // report mirror failure back to LP if we actually use a mirror
  90    string FailReason = LookupTag(Message, "FailReason");
  91    if(FailReason.size() != 0)
  92       ReportMirrorFailure(FailReason);
  93    else
  94       ReportMirrorFailure(ErrorText);
  95 }
  96                                                                         /*}}}*/
  97 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* Stash status and the file size. Note that setting Complete means
 100    sub-phases of the acquire process such as decompresion are operating */
 101 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 102 {
 103    Status = StatFetching;
 104    if (FileSize == 0 && Complete == false)
 105       FileSize = Size;
 106 }
 107                                                                         /*}}}*/
 108 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash,
 112                             pkgAcquire::MethodConfig *Cnf)
 113 {
 114    // We just downloaded something..
 115    string FileName = LookupTag(Message,"Filename");
 116    UsedMirror =  LookupTag(Message,"UsedMirror");
 117    if (Complete == false && !Local && FileName == DestFile)
 118    {
 119       if (Owner->Log != 0)
 120          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 121    }
 122
 123    if (FileSize == 0)
 124       FileSize= Size;
 125    Status = StatDone;
 126    ErrorText = string();
 127    Owner->Dequeue(this);
 128 }
 129                                                                         /*}}}*/
 130 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* This helper function is used by a lot of item methods as their final
 133    step */
 134 void pkgAcquire::Item::Rename(string From,string To)
 135 {
 136    if (rename(From.c_str(),To.c_str()) != 0)
 137    {
 138       char S[300];
 139       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 140               From.c_str(),To.c_str());
 141       Status = StatError;
 142       ErrorText = S;
 143    }
 144 }
 145                                                                         /*}}}*/
 146 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 147 {
 148    if(FileExists(DestFile))
 149       Rename(DestFile, DestFile + ".FAILED");
 150
 151    switch (error)
 152    {
 153       case HashSumMismatch:
 154          ErrorText = _("Hash Sum mismatch");
 155          Status = StatAuthError;
 156          ReportMirrorFailure("HashChecksumFailure");
 157          break;
 158       case SizeMismatch:
 159          ErrorText = _("Size mismatch");
 160          Status = StatAuthError;
 161          ReportMirrorFailure("SizeFailure");
 162          break;
 163       case InvalidFormat:
 164          ErrorText = _("Invalid file format");
 165          Status = StatError;
 166          // do not report as usually its not the mirrors fault, but Portal/Proxy
 167          break;
 168    }
 169    return false;
 170 }
 171                                                                         /*}}}*/
 172 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 173 // ---------------------------------------------------------------------
 174 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 175 {
 176    // we only act if a mirror was used at all
 177    if(UsedMirror.empty())
 178       return;
 179 #if 0
 180    std::cerr << "\nReportMirrorFailure: "
 181              << UsedMirror
 182              << " Uri: " << DescURI()
 183              << " FailCode: "
 184              << FailCode << std::endl;
 185 #endif
 186    const char *Args[40];
 187    unsigned int i = 0;
 188    string report = _config->Find("Methods::Mirror::ProblemReporting",
 189                                  "/usr/lib/apt/apt-report-mirror-failure");
 190    if(!FileExists(report))
 191       return;
 192    Args[i++] = report.c_str();
 193    Args[i++] = UsedMirror.c_str();
 194    Args[i++] = DescURI().c_str();
 195    Args[i++] = FailCode.c_str();
 196    Args[i++] = NULL;
 197    pid_t pid = ExecFork();
 198    if(pid < 0)
 199    {
 200       _error->Error("ReportMirrorFailure Fork failed");
 201       return;
 202    }
 203    else if(pid == 0)
 204    {
 205       execvp(Args[0], (char**)Args);
 206       std::cerr << "Could not exec " << Args[0] << std::endl;
 207       _exit(100);
 208    }
 209    if(!ExecWait(pid, "report-mirror-failure"))
 210    {
 211       _error->Warning("Couldn't report problem to '%s'",
 212                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 213    }
 214 }
 215                                                                         /*}}}*/
 216 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 217 // ---------------------------------------------------------------------
 218 /* Get a sub-index file based on checksums from a 'master' file and
 219    possibly query additional files */
 220 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 221                                  string const &URIDesc, string const &ShortDesc,
 222                                  HashString const &ExpectedHash)
 223    : Item(Owner), ExpectedHash(ExpectedHash)
 224 {
 225    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 226    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 227
 228    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 229    DestFile += URItoFileName(URI);
 230
 231    Desc.URI = URI;
 232    Desc.Description = URIDesc;
 233    Desc.Owner = this;
 234    Desc.ShortDesc = ShortDesc;
 235
 236    QueueURI(Desc);
 237
 238    if(Debug)
 239       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 240 }
 241                                                                         /*}}}*/
 242 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 243 // ---------------------------------------------------------------------
 244 /* The only header we use is the last-modified header. */
 245 string pkgAcqSubIndex::Custom600Headers()
 246 {
 247    string Final = _config->FindDir("Dir::State::lists");
 248    Final += URItoFileName(Desc.URI);
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true\nFail-Ignore: true\n";
 253    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 254 }
 255                                                                         /*}}}*/
 256 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)       /*{{{*/
 257 {
 258    if(Debug)
 259       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
 260
 261    Complete = false;
 262    Status = StatDone;
 263    Dequeue();
 264
 265    // No good Index is provided
 266 }
 267                                                                         /*}}}*/
 268 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 269                            pkgAcquire::MethodConfig *Cnf)
 270 {
 271    if(Debug)
 272       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 273
 274    string FileName = LookupTag(Message,"Filename");
 275    if (FileName.empty() == true)
 276    {
 277       Status = StatError;
 278       ErrorText = "Method gave a blank filename";
 279       return;
 280    }
 281
 282    if (FileName != DestFile)
 283    {
 284       Local = true;
 285       Desc.URI = "copy:" + FileName;
 286       QueueURI(Desc);
 287       return;
 288    }
 289
 290    Item::Done(Message,Size,Md5Hash,Cnf);
 291
 292    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 293
 294    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 295    indexRecords SubIndexParser;
 296    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 297       Status = StatError;
 298       ErrorText = SubIndexParser.ErrorText;
 299       return;
 300    }
 301
 302    // success in downloading the index
 303    // rename the index
 304    if(Debug)
 305       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 306    Rename(DestFile,FinalFile);
 307    chmod(FinalFile.c_str(),0644);
 308    DestFile = FinalFile;
 309
 310    if(ParseIndex(DestFile) == false)
 311       return Failed("", NULL);
 312
 313    Complete = true;
 314    Status = StatDone;
 315    Dequeue();
 316    return;
 317 }
 318                                                                         /*}}}*/
 319 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 320 {
 321    indexRecords SubIndexParser;
 322    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 323       return false;
 324    // so something with the downloaded index
 325    return true;
 326 }
 327                                                                         /*}}}*/
 328 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 329 // ---------------------------------------------------------------------
 330 /* Get the DiffIndex file first and see if there are patches available
 331  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 332  * patches. If anything goes wrong in that process, it will fall back to
 333  * the original packages file
 334  */
 335 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 336                                  string URI,string URIDesc,string ShortDesc,
 337                                  HashString ExpectedHash)
 338    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 339      Description(URIDesc)
 340 {
 341
 342    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 343
 344    Desc.Description = URIDesc + "/DiffIndex";
 345    Desc.Owner = this;
 346    Desc.ShortDesc = ShortDesc;
 347    Desc.URI = URI + ".diff/Index";
 348
 349    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 350    DestFile += URItoFileName(URI) + string(".DiffIndex");
 351
 352    if(Debug)
 353       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 354
 355    // look for the current package file
 356    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 357    CurrentPackagesFile += URItoFileName(RealURI);
 358
 359    // FIXME: this file:/ check is a hack to prevent fetching
 360    //        from local sources. this is really silly, and
 361    //        should be fixed cleanly as soon as possible
 362    if(!FileExists(CurrentPackagesFile) ||
 363       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 364    {
 365       // we don't have a pkg file or we don't want to queue
 366       if(Debug)
 367          std::clog << "No index file, local or canceld by user" << std::endl;
 368       Failed("", NULL);
 369       return;
 370    }
 371
 372    if(Debug)
 373       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 374          << CurrentPackagesFile << std::endl;
 375
 376    QueueURI(Desc);
 377
 378 }
 379                                                                         /*}}}*/
 380 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* The only header we use is the last-modified header. */
 383 string pkgAcqDiffIndex::Custom600Headers()
 384 {
 385    string Final = _config->FindDir("Dir::State::lists");
 386    Final += URItoFileName(RealURI) + string(".IndexDiff");
 387
 388    if(Debug)
 389       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 390
 391    struct stat Buf;
 392    if (stat(Final.c_str(),&Buf) != 0)
 393       return "\nIndex-File: true";
 394
 395    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 396 }
 397                                                                         /*}}}*/
 398 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 399 {
 400    if(Debug)
 401       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 402          << std::endl;
 403
 404    pkgTagSection Tags;
 405    string ServerSha1;
 406    vector<DiffInfo> available_patches;
 407
 408    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 409    pkgTagFile TF(&Fd);
 410    if (_error->PendingError() == true)
 411       return false;
 412
 413    if(TF.Step(Tags) == true)
 414    {
 415       bool found = false;
 416       DiffInfo d;
 417       string size;
 418
 419       string const tmp = Tags.FindS("SHA1-Current");
 420       std::stringstream ss(tmp);
 421       ss >> ServerSha1 >> size;
 422       unsigned long const ServerSize = atol(size.c_str());
 423
 424       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 425       SHA1Summation SHA1;
 426       SHA1.AddFD(fd);
 427       string const local_sha1 = SHA1.Result();
 428
 429       if(local_sha1 == ServerSha1)
 430       {
 431          // we have the same sha1 as the server so we are done here
 432          if(Debug)
 433             std::clog << "Package file is up-to-date" << std::endl;
 434          // list cleanup needs to know that this file as well as the already
 435          // present index is ours, so we create an empty diff to save it for us
 436          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 437                ExpectedHash, ServerSha1, available_patches);
 438          return true;
 439       }
 440       else
 441       {
 442          if(Debug)
 443             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 444
 445          // check the historie and see what patches we need
 446          string const history = Tags.FindS("SHA1-History");
 447          std::stringstream hist(history);
 448          while(hist >> d.sha1 >> size >> d.file)
 449          {
 450             // read until the first match is found
 451             // from that point on, we probably need all diffs
 452             if(d.sha1 == local_sha1)
 453                found=true;
 454             else if (found == false)
 455                continue;
 456
 457             if(Debug)
 458                std::clog << "Need to get diff: " << d.file << std::endl;
 459             available_patches.push_back(d);
 460          }
 461
 462          if (available_patches.empty() == false)
 463          {
 464             // patching with too many files is rather slow compared to a fast download
 465             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 466             if (fileLimit != 0 && fileLimit < available_patches.size())
 467             {
 468                if (Debug)
 469                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 470                         << ") so fallback to complete download" << std::endl;
 471                return false;
 472             }
 473
 474             // see if the patches are too big
 475             found = false; // it was true and it will be true again at the end
 476             d = *available_patches.begin();
 477             string const firstPatch = d.file;
 478             unsigned long patchesSize = 0;
 479             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 480             while(patches >> d.sha1 >> size >> d.file)
 481             {
 482                if (firstPatch == d.file)
 483                   found = true;
 484                else if (found == false)
 485                   continue;
 486
 487                patchesSize += atol(size.c_str());
 488             }
 489             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 490             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 491             {
 492                if (Debug)
 493                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 494                         << ") so fallback to complete download" << std::endl;
 495                return false;
 496             }
 497          }
 498       }
 499
 500       // we have something, queue the next diff
 501       if(found)
 502       {
 503          // queue the diffs
 504          string::size_type const last_space = Description.rfind(" ");
 505          if(last_space != string::npos)
 506             Description.erase(last_space, Description.size()-last_space);
 507
 508          /* decide if we should download patches one by one or in one go:
 509             The first is good if the server merges patches, but many don't so client
 510             based merging can be attempt in which case the second is better.
 511             "bad things" will happen if patches are merged on the server,
 512             but client side merging is attempt as well */
 513          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 514          if (pdiff_merge == true)
 515          {
 516             // reprepro adds this flag if it has merged patches on the server
 517             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 518             pdiff_merge = (precedence != "merged");
 519          }
 520
 521          if (pdiff_merge == false)
 522             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 523                   ExpectedHash, ServerSha1, available_patches);
 524          else
 525          {
 526             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 527             for(size_t i = 0; i < available_patches.size(); ++i)
 528                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 529                      available_patches[i], diffs);
 530          }
 531
 532          Complete = false;
 533          Status = StatDone;
 534          Dequeue();
 535          return true;
 536       }
 537    }
 538
 539    // Nothing found, report and return false
 540    // Failing here is ok, if we return false later, the full
 541    // IndexFile is queued
 542    if(Debug)
 543       std::clog << "Can't find a patch in the index file" << std::endl;
 544    return false;
 545 }
 546                                                                         /*}}}*/
 547 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 548 {
 549    if(Debug)
 550       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 551                 << "Falling back to normal index file acquire" << std::endl;
 552
 553    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 554                    ExpectedHash);
 555
 556    Complete = false;
 557    Status = StatDone;
 558    Dequeue();
 559 }
 560                                                                         /*}}}*/
 561 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 562                            pkgAcquire::MethodConfig *Cnf)
 563 {
 564    if(Debug)
 565       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 566
 567    Item::Done(Message,Size,Md5Hash,Cnf);
 568
 569    string FinalFile;
 570    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 571
 572    // success in downloading the index
 573    // rename the index
 574    FinalFile += string(".IndexDiff");
 575    if(Debug)
 576       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 577                 << std::endl;
 578    Rename(DestFile,FinalFile);
 579    chmod(FinalFile.c_str(),0644);
 580    DestFile = FinalFile;
 581
 582    if(!ParseDiffIndex(DestFile))
 583       return Failed("", NULL);
 584
 585    Complete = true;
 586    Status = StatDone;
 587    Dequeue();
 588    return;
 589 }
 590                                                                         /*}}}*/
 591 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* The package diff is added to the queue. one object is constructed
 594  * for each diff and the index
 595  */
 596 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 597                                    string URI,string URIDesc,string ShortDesc,
 598                                    HashString ExpectedHash,
 599                                    string ServerSha1,
 600                                    vector<DiffInfo> diffs)
 601    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 602      available_patches(diffs), ServerSha1(ServerSha1)
 603 {
 604
 605    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 606    DestFile += URItoFileName(URI);
 607
 608    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 609
 610    Description = URIDesc;
 611    Desc.Owner = this;
 612    Desc.ShortDesc = ShortDesc;
 613
 614    if(available_patches.empty() == true)
 615    {
 616       // we are done (yeah!)
 617       Finish(true);
 618    }
 619    else
 620    {
 621       // get the next diff
 622       State = StateFetchDiff;
 623       QueueNextDiff();
 624    }
 625 }
 626                                                                         /*}}}*/
 627 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 628 {
 629    if(Debug)
 630       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 631                 << "Falling back to normal index file acquire" << std::endl;
 632    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 633                    ExpectedHash);
 634    Finish();
 635 }
 636                                                                         /*}}}*/
 637 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 638 void pkgAcqIndexDiffs::Finish(bool allDone)
 639 {
 640    // we restore the original name, this is required, otherwise
 641    // the file will be cleaned
 642    if(allDone)
 643    {
 644       DestFile = _config->FindDir("Dir::State::lists");
 645       DestFile += URItoFileName(RealURI);
 646
 647       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 648       {
 649          RenameOnError(HashSumMismatch);
 650          Dequeue();
 651          return;
 652       }
 653
 654       // this is for the "real" finish
 655       Complete = true;
 656       Status = StatDone;
 657       Dequeue();
 658       if(Debug)
 659          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 660       return;
 661    }
 662
 663    if(Debug)
 664       std::clog << "Finishing: " << Desc.URI << std::endl;
 665    Complete = false;
 666    Status = StatDone;
 667    Dequeue();
 668    return;
 669 }
 670                                                                         /*}}}*/
 671 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 672 {
 673
 674    // calc sha1 of the just patched file
 675    string FinalFile = _config->FindDir("Dir::State::lists");
 676    FinalFile += URItoFileName(RealURI);
 677
 678    FileFd fd(FinalFile, FileFd::ReadOnly);
 679    SHA1Summation SHA1;
 680    SHA1.AddFD(fd);
 681    string local_sha1 = string(SHA1.Result());
 682    if(Debug)
 683       std::clog << "QueueNextDiff: "
 684                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 685
 686    // final file reached before all patches are applied
 687    if(local_sha1 == ServerSha1)
 688    {
 689       Finish(true);
 690       return true;
 691    }
 692
 693    // remove all patches until the next matching patch is found
 694    // this requires the Index file to be ordered
 695    for(vector<DiffInfo>::iterator I=available_patches.begin();
 696        available_patches.empty() == false &&
 697           I != available_patches.end() &&
 698           I->sha1 != local_sha1;
 699        ++I)
 700    {
 701       available_patches.erase(I);
 702    }
 703
 704    // error checking and falling back if no patch was found
 705    if(available_patches.empty() == true)
 706    {
 707       Failed("", NULL);
 708       return false;
 709    }
 710
 711    // queue the right diff
 712    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 713    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 714    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 715    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 716
 717    if(Debug)
 718       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 719
 720    QueueURI(Desc);
 721
 722    return true;
 723 }
 724                                                                         /*}}}*/
 725 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 726                             pkgAcquire::MethodConfig *Cnf)
 727 {
 728    if(Debug)
 729       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 730
 731    Item::Done(Message,Size,Md5Hash,Cnf);
 732
 733    string FinalFile;
 734    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 735
 736    // success in downloading a diff, enter ApplyDiff state
 737    if(State == StateFetchDiff)
 738    {
 739
 740       // rred excepts the patch as $FinalFile.ed
 741       Rename(DestFile,FinalFile+".ed");
 742
 743       if(Debug)
 744          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 745
 746       State = StateApplyDiff;
 747       Local = true;
 748       Desc.URI = "rred:" + FinalFile;
 749       QueueURI(Desc);
 750       Mode = "rred";
 751       return;
 752    }
 753
 754
 755    // success in download/apply a diff, queue next (if needed)
 756    if(State == StateApplyDiff)
 757    {
 758       // remove the just applied patch
 759       available_patches.erase(available_patches.begin());
 760       unlink((FinalFile + ".ed").c_str());
 761
 762       // move into place
 763       if(Debug)
 764       {
 765          std::clog << "Moving patched file in place: " << std::endl
 766                    << DestFile << " -> " << FinalFile << std::endl;
 767       }
 768       Rename(DestFile,FinalFile);
 769       chmod(FinalFile.c_str(),0644);
 770
 771       // see if there is more to download
 772       if(available_patches.empty() == false) {
 773          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 774                               ExpectedHash, ServerSha1, available_patches);
 775          return Finish();
 776       } else
 777          return Finish(true);
 778    }
 779 }
 780                                                                         /*}}}*/
 781 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 782 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 783                                    string const &URI, string const &URIDesc,
 784                                    string const &ShortDesc, HashString const &ExpectedHash,
 785                                    DiffInfo const &patch,
 786                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 787    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 788      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 789 {
 790
 791    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 792    DestFile += URItoFileName(URI);
 793
 794    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 795
 796    Description = URIDesc;
 797    Desc.Owner = this;
 798    Desc.ShortDesc = ShortDesc;
 799
 800    Desc.URI = string(RealURI) + ".diff/" + patch.file + ".gz";
 801    Desc.Description = Description + " " + patch.file + string(".pdiff");
 802    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 803    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 804
 805    if(Debug)
 806       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 807
 808    QueueURI(Desc);
 809 }
 810                                                                         /*}}}*/
 811 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 812 {
 813    if(Debug)
 814       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 815    Complete = false;
 816    Status = StatDone;
 817    Dequeue();
 818
 819    // check if we are the first to fail, otherwise we are done here
 820    State = StateDoneDiff;
 821    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 822          I != allPatches->end(); ++I)
 823       if ((*I)->State == StateErrorDiff)
 824          return;
 825
 826    // first failure means we should fallback
 827    State = StateErrorDiff;
 828    std::clog << "Falling back to normal index file acquire" << std::endl;
 829    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 830                    ExpectedHash);
 831 }
 832                                                                         /*}}}*/
 833 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 834                             pkgAcquire::MethodConfig *Cnf)
 835 {
 836    if(Debug)
 837       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 838
 839    Item::Done(Message,Size,Md5Hash,Cnf);
 840
 841    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 842
 843    if (State == StateFetchDiff)
 844    {
 845       // rred expects the patch as $FinalFile.ed.$patchname.gz
 846       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 847
 848       // check if this is the last completed diff
 849       State = StateDoneDiff;
 850       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 851             I != allPatches->end(); ++I)
 852          if ((*I)->State != StateDoneDiff)
 853          {
 854             if(Debug)
 855                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 856             return;
 857          }
 858
 859       // this is the last completed diff, so we are ready to apply now
 860       State = StateApplyDiff;
 861
 862       if(Debug)
 863          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 864
 865       Local = true;
 866       Desc.URI = "rred:" + FinalFile;
 867       QueueURI(Desc);
 868       Mode = "rred";
 869       return;
 870    }
 871    // success in download/apply all diffs, clean up
 872    else if (State == StateApplyDiff)
 873    {
 874       // see if we really got the expected file
 875       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 876       {
 877          RenameOnError(HashSumMismatch);
 878          return;
 879       }
 880
 881       // move the result into place
 882       if(Debug)
 883          std::clog << "Moving patched file in place: " << std::endl
 884                    << DestFile << " -> " << FinalFile << std::endl;
 885       Rename(DestFile, FinalFile);
 886       chmod(FinalFile.c_str(), 0644);
 887
 888       // otherwise lists cleanup will eat the file
 889       DestFile = FinalFile;
 890
 891       // ensure the ed's are gone regardless of list-cleanup
 892       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 893             I != allPatches->end(); ++I)
 894       {
 895             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 896             unlink(patch.c_str());
 897       }
 898
 899       // all set and done
 900       Complete = true;
 901       if(Debug)
 902          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 903    }
 904 }
 905                                                                         /*}}}*/
 906 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 907 // ---------------------------------------------------------------------
 908 /* The package file is added to the queue and a second class is
 909    instantiated to fetch the revision file */
 910 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 911                          string URI,string URIDesc,string ShortDesc,
 912                          HashString ExpectedHash, string comprExt)
 913    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 914 {
 915    if(comprExt.empty() == true)
 916    {
 917       // autoselect the compression method
 918       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 919       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 920          comprExt.append(*t).append(" ");
 921       if (comprExt.empty() == false)
 922          comprExt.erase(comprExt.end()-1);
 923    }
 924    CompressionExtension = comprExt;
 925
 926    Init(URI, URIDesc, ShortDesc);
 927 }
 928 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 929                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 930    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 931 {
 932    // autoselect the compression method
 933    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 934    CompressionExtension = "";
 935    if (ExpectedHash.empty() == false)
 936    {
 937       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 938          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 939             CompressionExtension.append(*t).append(" ");
 940    }
 941    else
 942    {
 943       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 944          CompressionExtension.append(*t).append(" ");
 945    }
 946    if (CompressionExtension.empty() == false)
 947       CompressionExtension.erase(CompressionExtension.end()-1);
 948
 949    // only verify non-optional targets, see acquire-item.h for a FIXME
 950    // to make this more flexible
 951    if (Target->IsOptional())
 952      Verify = false;
 953    else
 954      Verify = true;
 955
 956    Init(Target->URI, Target->Description, Target->ShortDesc);
 957 }
 958                                                                         /*}}}*/
 959 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 960 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 961    Decompression = false;
 962    Erase = false;
 963
 964    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 965    DestFile += URItoFileName(URI);
 966
 967    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 968    if (comprExt == "uncompressed")
 969       Desc.URI = URI;
 970    else
 971       Desc.URI = URI + '.' + comprExt;
 972
 973    Desc.Description = URIDesc;
 974    Desc.Owner = this;
 975    Desc.ShortDesc = ShortDesc;
 976
 977    QueueURI(Desc);
 978 }
 979                                                                         /*}}}*/
 980 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 981 // ---------------------------------------------------------------------
 982 /* The only header we use is the last-modified header. */
 983 string pkgAcqIndex::Custom600Headers()
 984 {
 985    string Final = _config->FindDir("Dir::State::lists");
 986    Final += URItoFileName(RealURI);
 987    if (_config->FindB("Acquire::GzipIndexes",false))
 988       Final += ".gz";
 989
 990    string msg = "\nIndex-File: true";
 991    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 992    //        seems to be difficult without breaking ABI
 993    if (ShortDesc().find("Translation") != 0)
 994       msg += "\nFail-Ignore: true";
 995    struct stat Buf;
 996    if (stat(Final.c_str(),&Buf) == 0)
 997       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 998
 999    return msg;
1000 }
1001                                                                         /*}}}*/
1002 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1003 {
1004    size_t const nextExt = CompressionExtension.find(' ');
1005    if (nextExt != std::string::npos)
1006    {
1007       CompressionExtension = CompressionExtension.substr(nextExt+1);
1008       Init(RealURI, Desc.Description, Desc.ShortDesc);
1009       return;
1010    }
1011
1012    // on decompression failure, remove bad versions in partial/
1013    if (Decompression && Erase) {
1014       string s = _config->FindDir("Dir::State::lists") + "partial/";
1015       s.append(URItoFileName(RealURI));
1016       unlink(s.c_str());
1017    }
1018
1019    Item::Failed(Message,Cnf);
1020 }
1021                                                                         /*}}}*/
1022 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1023 // ---------------------------------------------------------------------
1024 /* This goes through a number of states.. On the initial fetch the
1025    method could possibly return an alternate filename which points
1026    to the uncompressed version of the file. If this is so the file
1027    is copied into the partial directory. In all other cases the file
1028    is decompressed with a gzip uri. */
1029 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1030                        pkgAcquire::MethodConfig *Cfg)
1031 {
1032    Item::Done(Message,Size,Hash,Cfg);
1033
1034    if (Decompression == true)
1035    {
1036       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1037       {
1038          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1039          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1040       }
1041
1042       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1043       {
1044          RenameOnError(HashSumMismatch);
1045          return;
1046       }
1047
1048       /* Verify the index file for correctness (all indexes must
1049        * have a Package field) (LP: #346386) (Closes: #627642) */
1050       if (Verify == true)
1051       {
1052          FileFd fd(DestFile, FileFd::ReadOnly);
1053          // Only test for correctness if the file is not empty (empty is ok)
1054          if (fd.FileSize() > 0)
1055          {
1056             pkgTagSection sec;
1057             pkgTagFile tag(&fd);
1058
1059             // all our current indexes have a field 'Package' in each section
1060             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1061             {
1062                RenameOnError(InvalidFormat);
1063                return;
1064             }
1065          }
1066       }
1067
1068       // Done, move it into position
1069       string FinalFile = _config->FindDir("Dir::State::lists");
1070       FinalFile += URItoFileName(RealURI);
1071       Rename(DestFile,FinalFile);
1072       chmod(FinalFile.c_str(),0644);
1073
1074       /* We restore the original name to DestFile so that the clean operation
1075          will work OK */
1076       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1077       DestFile += URItoFileName(RealURI);
1078
1079       // Remove the compressed version.
1080       if (Erase == true)
1081          unlink(DestFile.c_str());
1082       return;
1083    }
1084
1085    Erase = false;
1086    Complete = true;
1087
1088    // Handle the unzipd case
1089    string FileName = LookupTag(Message,"Alt-Filename");
1090    if (FileName.empty() == false)
1091    {
1092       // The files timestamp matches
1093       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1094          return;
1095       Decompression = true;
1096       Local = true;
1097       DestFile += ".decomp";
1098       Desc.URI = "copy:" + FileName;
1099       QueueURI(Desc);
1100       Mode = "copy";
1101       return;
1102    }
1103
1104    FileName = LookupTag(Message,"Filename");
1105    if (FileName.empty() == true)
1106    {
1107       Status = StatError;
1108       ErrorText = "Method gave a blank filename";
1109    }
1110
1111    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1112
1113    // The files timestamp matches
1114    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1115        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1116           // Update DestFile for .gz suffix so that the clean operation keeps it
1117           DestFile += ".gz";
1118       return;
1119     }
1120
1121    if (FileName == DestFile)
1122       Erase = true;
1123    else
1124       Local = true;
1125
1126    string decompProg;
1127
1128    // If we enable compressed indexes and already have gzip, keep it
1129    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1130       string FinalFile = _config->FindDir("Dir::State::lists");
1131       FinalFile += URItoFileName(RealURI) + ".gz";
1132       Rename(DestFile,FinalFile);
1133       chmod(FinalFile.c_str(),0644);
1134
1135       // Update DestFile for .gz suffix so that the clean operation keeps it
1136       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1137       DestFile += URItoFileName(RealURI) + ".gz";
1138       return;
1139     }
1140
1141    // get the binary name for your used compression type
1142    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1143    if(decompProg.empty() == false);
1144    else if(compExt == "uncompressed")
1145       decompProg = "copy";
1146    else {
1147       _error->Error("Unsupported extension: %s", compExt.c_str());
1148       return;
1149    }
1150
1151    Decompression = true;
1152    DestFile += ".decomp";
1153    Desc.URI = decompProg + ":" + FileName;
1154    QueueURI(Desc);
1155
1156    // FIXME: this points to a c++ string that goes out of scope
1157    Mode = decompProg.c_str();
1158 }
1159                                                                         /*}}}*/
1160 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1161 // ---------------------------------------------------------------------
1162 /* The Translation file is added to the queue */
1163 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1164                             string URI,string URIDesc,string ShortDesc)
1165   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1166 {
1167 }
1168 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1169                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1170   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1171 {
1172 }
1173                                                                         /*}}}*/
1174 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1175 // ---------------------------------------------------------------------
1176 string pkgAcqIndexTrans::Custom600Headers()
1177 {
1178    string Final = _config->FindDir("Dir::State::lists");
1179    Final += URItoFileName(RealURI);
1180
1181    struct stat Buf;
1182    if (stat(Final.c_str(),&Buf) != 0)
1183       return "\nFail-Ignore: true\nIndex-File: true";
1184    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1185 }
1186                                                                         /*}}}*/
1187 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1188 // ---------------------------------------------------------------------
1189 /* */
1190 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1191 {
1192    size_t const nextExt = CompressionExtension.find(' ');
1193    if (nextExt != std::string::npos)
1194    {
1195       CompressionExtension = CompressionExtension.substr(nextExt+1);
1196       Init(RealURI, Desc.Description, Desc.ShortDesc);
1197       Status = StatIdle;
1198       return;
1199    }
1200
1201    if (Cnf->LocalOnly == true ||
1202        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1203    {
1204       // Ignore this
1205       Status = StatDone;
1206       Complete = false;
1207       Dequeue();
1208       return;
1209    }
1210
1211    Item::Failed(Message,Cnf);
1212 }
1213                                                                         /*}}}*/
1214 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1215                              string URI,string URIDesc,string ShortDesc,
1216                              string MetaIndexURI, string MetaIndexURIDesc,
1217                              string MetaIndexShortDesc,
1218                              const vector<IndexTarget*>* IndexTargets,
1219                              indexRecords* MetaIndexParser) :
1220    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1221    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1222    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1223 {
1224    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1225    DestFile += URItoFileName(URI);
1226
1227    // remove any partial downloaded sig-file in partial/.
1228    // it may confuse proxies and is too small to warrant a
1229    // partial download anyway
1230    unlink(DestFile.c_str());
1231
1232    // Create the item
1233    Desc.Description = URIDesc;
1234    Desc.Owner = this;
1235    Desc.ShortDesc = ShortDesc;
1236    Desc.URI = URI;
1237
1238    string Final = _config->FindDir("Dir::State::lists");
1239    Final += URItoFileName(RealURI);
1240    if (RealFileExists(Final) == true)
1241    {
1242       // File was already in place.  It needs to be re-downloaded/verified
1243       // because Release might have changed, we do give it a different
1244       // name than DestFile because otherwise the http method will
1245       // send If-Range requests and there are too many broken servers
1246       // out there that do not understand them
1247       LastGoodSig = DestFile+".reverify";
1248       Rename(Final,LastGoodSig);
1249    }
1250
1251    QueueURI(Desc);
1252 }
1253                                                                         /*}}}*/
1254 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1255 {
1256    // if the file was never queued undo file-changes done in the constructor
1257    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1258          LastGoodSig.empty() == false)
1259    {
1260       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1261       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1262          Rename(LastGoodSig, Final);
1263    }
1264
1265 }
1266                                                                         /*}}}*/
1267 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1268 // ---------------------------------------------------------------------
1269 /* The only header we use is the last-modified header. */
1270 string pkgAcqMetaSig::Custom600Headers()
1271 {
1272    struct stat Buf;
1273    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1274       return "\nIndex-File: true";
1275
1276    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1277 }
1278
1279 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1280                          pkgAcquire::MethodConfig *Cfg)
1281 {
1282    Item::Done(Message,Size,MD5,Cfg);
1283
1284    string FileName = LookupTag(Message,"Filename");
1285    if (FileName.empty() == true)
1286    {
1287       Status = StatError;
1288       ErrorText = "Method gave a blank filename";
1289       return;
1290    }
1291
1292    if (FileName != DestFile)
1293    {
1294       // We have to copy it into place
1295       Local = true;
1296       Desc.URI = "copy:" + FileName;
1297       QueueURI(Desc);
1298       return;
1299    }
1300
1301    Complete = true;
1302
1303    // put the last known good file back on i-m-s hit (it will
1304    // be re-verified again)
1305    // Else do nothing, we have the new file in DestFile then
1306    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1307       Rename(LastGoodSig, DestFile);
1308
1309    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1310    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1311                        MetaIndexShortDesc,  DestFile, IndexTargets,
1312                        MetaIndexParser);
1313
1314 }
1315                                                                         /*}}}*/
1316 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1317 {
1318    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1319
1320    // if we get a network error we fail gracefully
1321    if(Status == StatTransientNetworkError)
1322    {
1323       Item::Failed(Message,Cnf);
1324       // move the sigfile back on transient network failures
1325       if(FileExists(LastGoodSig))
1326          Rename(LastGoodSig,Final);
1327
1328       // set the status back to , Item::Failed likes to reset it
1329       Status = pkgAcquire::Item::StatTransientNetworkError;
1330       return;
1331    }
1332
1333    // Delete any existing sigfile when the acquire failed
1334    unlink(Final.c_str());
1335
1336    // queue a pkgAcqMetaIndex with no sigfile
1337    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1338                        "", IndexTargets, MetaIndexParser);
1339
1340    if (Cnf->LocalOnly == true ||
1341        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1342    {
1343       // Ignore this
1344       Status = StatDone;
1345       Complete = false;
1346       Dequeue();
1347       return;
1348    }
1349
1350    Item::Failed(Message,Cnf);
1351 }
1352                                                                         /*}}}*/
1353 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1354                                  string URI,string URIDesc,string ShortDesc,
1355                                  string SigFile,
1356                                  const vector<struct IndexTarget*>* IndexTargets,
1357                                  indexRecords* MetaIndexParser) :
1358    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1359    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1360 {
1361    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1362    DestFile += URItoFileName(URI);
1363
1364    // Create the item
1365    Desc.Description = URIDesc;
1366    Desc.Owner = this;
1367    Desc.ShortDesc = ShortDesc;
1368    Desc.URI = URI;
1369
1370    QueueURI(Desc);
1371 }
1372                                                                         /*}}}*/
1373 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1374 // ---------------------------------------------------------------------
1375 /* The only header we use is the last-modified header. */
1376 string pkgAcqMetaIndex::Custom600Headers()
1377 {
1378    string Final = _config->FindDir("Dir::State::lists");
1379    Final += URItoFileName(RealURI);
1380
1381    struct stat Buf;
1382    if (stat(Final.c_str(),&Buf) != 0)
1383       return "\nIndex-File: true";
1384
1385    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1386 }
1387                                                                         /*}}}*/
1388 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1389                            pkgAcquire::MethodConfig *Cfg)
1390 {
1391    Item::Done(Message,Size,Hash,Cfg);
1392
1393    // MetaIndexes are done in two passes: one to download the
1394    // metaindex with an appropriate method, and a second to verify it
1395    // with the gpgv method
1396
1397    if (AuthPass == true)
1398    {
1399       AuthDone(Message);
1400
1401       // all cool, move Release file into place
1402       Complete = true;
1403    }
1404    else
1405    {
1406       RetrievalDone(Message);
1407       if (!Complete)
1408          // Still more retrieving to do
1409          return;
1410
1411       if (SigFile == "")
1412       {
1413          // There was no signature file, so we are finished.  Download
1414          // the indexes and do only hashsum verification if possible
1415          MetaIndexParser->Load(DestFile);
1416          QueueIndexes(false);
1417       }
1418       else
1419       {
1420          // There was a signature file, so pass it to gpgv for
1421          // verification
1422
1423          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1424             std::cerr << "Metaindex acquired, queueing gpg verification ("
1425                       << SigFile << "," << DestFile << ")\n";
1426          AuthPass = true;
1427          Desc.URI = "gpgv:" + SigFile;
1428          QueueURI(Desc);
1429          Mode = "gpgv";
1430          return;
1431       }
1432    }
1433
1434    if (Complete == true)
1435    {
1436       string FinalFile = _config->FindDir("Dir::State::lists");
1437       FinalFile += URItoFileName(RealURI);
1438       if (SigFile == DestFile)
1439          SigFile = FinalFile;
1440       Rename(DestFile,FinalFile);
1441       chmod(FinalFile.c_str(),0644);
1442       DestFile = FinalFile;
1443    }
1444 }
1445                                                                         /*}}}*/
1446 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1447 {
1448    // We have just finished downloading a Release file (it is not
1449    // verified yet)
1450
1451    string FileName = LookupTag(Message,"Filename");
1452    if (FileName.empty() == true)
1453    {
1454       Status = StatError;
1455       ErrorText = "Method gave a blank filename";
1456       return;
1457    }
1458
1459    if (FileName != DestFile)
1460    {
1461       Local = true;
1462       Desc.URI = "copy:" + FileName;
1463       QueueURI(Desc);
1464       return;
1465    }
1466
1467    // make sure to verify against the right file on I-M-S hit
1468    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1469    if(IMSHit)
1470    {
1471       string FinalFile = _config->FindDir("Dir::State::lists");
1472       FinalFile += URItoFileName(RealURI);
1473       if (SigFile == DestFile)
1474       {
1475          SigFile = FinalFile;
1476          // constructor of pkgAcqMetaClearSig moved it out of the way,
1477          // now move it back in on IMS hit for the 'old' file
1478          string const OldClearSig = DestFile + ".reverify";
1479          if (RealFileExists(OldClearSig) == true)
1480             Rename(OldClearSig, FinalFile);
1481       }
1482       DestFile = FinalFile;
1483    }
1484    Complete = true;
1485 }
1486                                                                         /*}}}*/
1487 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1488 {
1489    // At this point, the gpgv method has succeeded, so there is a
1490    // valid signature from a key in the trusted keyring.  We
1491    // perform additional verification of its contents, and use them
1492    // to verify the indexes we are about to download
1493
1494    if (!MetaIndexParser->Load(DestFile))
1495    {
1496       Status = StatAuthError;
1497       ErrorText = MetaIndexParser->ErrorText;
1498       return;
1499    }
1500
1501    if (!VerifyVendor(Message))
1502    {
1503       return;
1504    }
1505
1506    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1507       std::cerr << "Signature verification succeeded: "
1508                 << DestFile << std::endl;
1509
1510    // Download further indexes with verification
1511    QueueIndexes(true);
1512
1513    // is it a clearsigned MetaIndex file?
1514    if (DestFile == SigFile)
1515       return;
1516
1517    // Done, move signature file into position
1518    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1519       URItoFileName(RealURI) + ".gpg";
1520    Rename(SigFile,VerifiedSigFile);
1521    chmod(VerifiedSigFile.c_str(),0644);
1522 }
1523                                                                         /*}}}*/
1524 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1525 {
1526 #if 0
1527    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1528     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1529    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1530    {
1531       Status = StatError;
1532       ErrorText = MetaIndexParser->ErrorText;
1533       return;
1534    }
1535 #endif
1536    bool transInRelease = false;
1537    {
1538       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1539       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1540          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1541          if (k->find("Translation-") != std::string::npos)
1542          {
1543             transInRelease = true;
1544             break;
1545          }
1546    }
1547
1548    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1549         Target != IndexTargets->end();
1550         ++Target)
1551    {
1552       HashString ExpectedIndexHash;
1553       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1554       bool compressedAvailable = false;
1555       if (Record == NULL)
1556       {
1557          if ((*Target)->IsOptional() == true)
1558          {
1559             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1560             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1561                if (MetaIndexParser->Exists(string((*Target)->MetaKey).append(".").append(*t)) == true)
1562                {
1563                   compressedAvailable = true;
1564                   break;
1565                }
1566          }
1567          else if (verify == true)
1568          {
1569             Status = StatAuthError;
1570             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1571             return;
1572          }
1573       }
1574       else
1575       {
1576          ExpectedIndexHash = Record->Hash;
1577          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1578          {
1579             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1580             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1581             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1582          }
1583          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1584          {
1585             Status = StatAuthError;
1586             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1587             return;
1588          }
1589       }
1590
1591       if ((*Target)->IsOptional() == true)
1592       {
1593          if ((*Target)->IsSubIndex() == true)
1594             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1595                                 (*Target)->ShortDesc, ExpectedIndexHash);
1596          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1597          {
1598             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1599                 MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true)
1600                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1601                                    (*Target)->ShortDesc, ExpectedIndexHash);
1602             else
1603                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1604          }
1605          continue;
1606       }
1607
1608       /* Queue Packages file (either diff or full packages files, depending
1609          on the users option) - we also check if the PDiff Index file is listed
1610          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1611          instead, but passing the required info to it is to much hassle */
1612       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1613           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1614          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1615                              (*Target)->ShortDesc, ExpectedIndexHash);
1616       else
1617          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1618    }
1619 }
1620                                                                         /*}}}*/
1621 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1622 {
1623    string::size_type pos;
1624
1625    // check for missing sigs (that where not fatal because otherwise we had
1626    // bombed earlier)
1627    string missingkeys;
1628    string msg = _("There is no public key available for the "
1629                   "following key IDs:\n");
1630    pos = Message.find("NO_PUBKEY ");
1631    if (pos != std::string::npos)
1632    {
1633       string::size_type start = pos+strlen("NO_PUBKEY ");
1634       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1635       missingkeys += (Fingerprint);
1636    }
1637    if(!missingkeys.empty())
1638       _error->Warning("%s", string(msg+missingkeys).c_str());
1639
1640    string Transformed = MetaIndexParser->GetExpectedDist();
1641
1642    if (Transformed == "../project/experimental")
1643    {
1644       Transformed = "experimental";
1645    }
1646
1647    pos = Transformed.rfind('/');
1648    if (pos != string::npos)
1649    {
1650       Transformed = Transformed.substr(0, pos);
1651    }
1652
1653    if (Transformed == ".")
1654    {
1655       Transformed = "";
1656    }
1657
1658    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1659        MetaIndexParser->GetValidUntil() > 0) {
1660       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1661       if (invalid_since > 0)
1662          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1663          // the time since then the file is invalid - formated in the same way as in
1664          // the download progress display (e.g. 7d 3h 42min 1s)
1665          return _error->Error(
1666             _("Release file for %s is expired (invalid since %s). "
1667               "Updates for this repository will not be applied."),
1668             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1669    }
1670
1671    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1672    {
1673       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1674       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1675       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1676    }
1677
1678    if (MetaIndexParser->CheckDist(Transformed) == false)
1679    {
1680       // This might become fatal one day
1681 //       Status = StatAuthError;
1682 //       ErrorText = "Conflicting distribution; expected "
1683 //          + MetaIndexParser->GetExpectedDist() + " but got "
1684 //          + MetaIndexParser->GetDist();
1685 //       return false;
1686       if (!Transformed.empty())
1687       {
1688          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1689                          Desc.Description.c_str(),
1690                          Transformed.c_str(),
1691                          MetaIndexParser->GetDist().c_str());
1692       }
1693    }
1694
1695    return true;
1696 }
1697                                                                         /*}}}*/
1698 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1699 // ---------------------------------------------------------------------
1700 /* */
1701 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1702 {
1703    if (AuthPass == true)
1704    {
1705       // gpgv method failed, if we have a good signature
1706       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1707       if (DestFile != SigFile)
1708          LastGoodSigFile.append(".gpg");
1709       LastGoodSigFile.append(".reverify");
1710
1711       if(FileExists(LastGoodSigFile))
1712       {
1713          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1714          if (DestFile != SigFile)
1715             VerifiedSigFile.append(".gpg");
1716          Rename(LastGoodSigFile, VerifiedSigFile);
1717          Status = StatTransientNetworkError;
1718          _error->Warning(_("An error occurred during the signature "
1719                            "verification. The repository is not updated "
1720                            "and the previous index files will be used. "
1721                            "GPG error: %s: %s\n"),
1722                          Desc.Description.c_str(),
1723                          LookupTag(Message,"Message").c_str());
1724          RunScripts("APT::Update::Auth-Failure");
1725          return;
1726       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1727          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1728          _error->Error(_("GPG error: %s: %s"),
1729                          Desc.Description.c_str(),
1730                          LookupTag(Message,"Message").c_str());
1731          return;
1732       } else {
1733          _error->Warning(_("GPG error: %s: %s"),
1734                          Desc.Description.c_str(),
1735                          LookupTag(Message,"Message").c_str());
1736       }
1737       // gpgv method failed
1738       ReportMirrorFailure("GPGFailure");
1739    }
1740
1741    /* Always move the meta index, even if gpgv failed. This ensures
1742     * that PackageFile objects are correctly filled in */
1743    if (FileExists(DestFile)) {
1744       string FinalFile = _config->FindDir("Dir::State::lists");
1745       FinalFile += URItoFileName(RealURI);
1746       /* InRelease files become Release files, otherwise
1747        * they would be considered as trusted later on */
1748       if (SigFile == DestFile) {
1749          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1750                                        "Release");
1751          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1752                                        "Release");
1753          SigFile = FinalFile;
1754       }
1755       Rename(DestFile,FinalFile);
1756       chmod(FinalFile.c_str(),0644);
1757
1758       DestFile = FinalFile;
1759    }
1760
1761    // No Release file was present, or verification failed, so fall
1762    // back to queueing Packages files without verification
1763    QueueIndexes(false);
1764 }
1765                                                                         /*}}}*/
1766 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1767                 string const &URI, string const &URIDesc, string const &ShortDesc,
1768                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1769                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1770                 const vector<struct IndexTarget*>* IndexTargets,
1771                 indexRecords* MetaIndexParser) :
1772         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1773         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1774         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1775 {
1776    SigFile = DestFile;
1777
1778    // keep the old InRelease around in case of transistent network errors
1779    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1780    if (RealFileExists(Final) == true)
1781    {
1782       string const LastGoodSig = DestFile + ".reverify";
1783       Rename(Final,LastGoodSig);
1784    }
1785 }
1786                                                                         /*}}}*/
1787 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1788 {
1789    // if the file was never queued undo file-changes done in the constructor
1790    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1791    {
1792       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1793       string const LastGoodSig = DestFile + ".reverify";
1794       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1795          Rename(LastGoodSig, Final);
1796    }
1797 }
1798                                                                         /*}}}*/
1799 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1800 // ---------------------------------------------------------------------
1801 // FIXME: this can go away once the InRelease file is used widely
1802 string pkgAcqMetaClearSig::Custom600Headers()
1803 {
1804    string Final = _config->FindDir("Dir::State::lists");
1805    Final += URItoFileName(RealURI);
1806
1807    struct stat Buf;
1808    if (stat(Final.c_str(),&Buf) != 0)
1809    {
1810       Final = DestFile + ".reverify";
1811       if (stat(Final.c_str(),&Buf) != 0)
1812          return "\nIndex-File: true\nFail-Ignore: true\n";
1813    }
1814
1815    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1816 }
1817                                                                         /*}}}*/
1818 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1819 {
1820    if (AuthPass == false)
1821    {
1822       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1823       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1824       string FinalFile = _config->FindDir("Dir::State::lists");
1825       FinalFile.append(URItoFileName(RealURI));
1826       if (FileExists(FinalFile))
1827          unlink(FinalFile.c_str());
1828
1829       new pkgAcqMetaSig(Owner,
1830                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1831                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1832                         IndexTargets, MetaIndexParser);
1833       if (Cnf->LocalOnly == true ||
1834           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1835          Dequeue();
1836    }
1837    else
1838       pkgAcqMetaIndex::Failed(Message, Cnf);
1839 }
1840                                                                         /*}}}*/
1841 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1842 // ---------------------------------------------------------------------
1843 /* This just sets up the initial fetch environment and queues the first
1844    possibilitiy */
1845 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1846                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1847                              string &StoreFilename) :
1848                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1849                StoreFilename(StoreFilename), Vf(Version.FileList()),
1850                Trusted(false)
1851 {
1852    Retries = _config->FindI("Acquire::Retries",0);
1853
1854    if (Version.Arch() == 0)
1855    {
1856       _error->Error(_("I wasn't able to locate a file for the %s package. "
1857                       "This might mean you need to manually fix this package. "
1858                       "(due to missing arch)"),
1859                     Version.ParentPkg().Name());
1860       return;
1861    }
1862
1863    /* We need to find a filename to determine the extension. We make the
1864       assumption here that all the available sources for this version share
1865       the same extension.. */
1866    // Skip not source sources, they do not have file fields.
1867    for (; Vf.end() == false; ++Vf)
1868    {
1869       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1870          continue;
1871       break;
1872    }
1873
1874    // Does not really matter here.. we are going to fail out below
1875    if (Vf.end() != true)
1876    {
1877       // If this fails to get a file name we will bomb out below.
1878       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1879       if (_error->PendingError() == true)
1880          return;
1881
1882       // Generate the final file name as: package_version_arch.foo
1883       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1884                       QuoteString(Version.VerStr(),"_:") + '_' +
1885                       QuoteString(Version.Arch(),"_:.") +
1886                       "." + flExtension(Parse.FileName());
1887    }
1888
1889    // check if we have one trusted source for the package. if so, switch
1890    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1891    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1892    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1893    bool seenUntrusted = false;
1894    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1895    {
1896       pkgIndexFile *Index;
1897       if (Sources->FindIndex(i.File(),Index) == false)
1898          continue;
1899
1900       if (debugAuth == true)
1901          std::cerr << "Checking index: " << Index->Describe()
1902                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1903
1904       if (Index->IsTrusted() == true)
1905       {
1906          Trusted = true;
1907          if (allowUnauth == false)
1908             break;
1909       }
1910       else
1911          seenUntrusted = true;
1912    }
1913
1914    // "allow-unauthenticated" restores apts old fetching behaviour
1915    // that means that e.g. unauthenticated file:// uris are higher
1916    // priority than authenticated http:// uris
1917    if (allowUnauth == true && seenUntrusted == true)
1918       Trusted = false;
1919
1920    // Select a source
1921    if (QueueNext() == false && _error->PendingError() == false)
1922       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1923                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1924 }
1925                                                                         /*}}}*/
1926 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1927 // ---------------------------------------------------------------------
1928 /* This queues the next available file version for download. It checks if
1929    the archive is already available in the cache and stashs the MD5 for
1930    checking later. */
1931 bool pkgAcqArchive::QueueNext()
1932 {
1933    string const ForceHash = _config->Find("Acquire::ForceHash");
1934    for (; Vf.end() == false; ++Vf)
1935    {
1936       // Ignore not source sources
1937       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1938          continue;
1939
1940       // Try to cross match against the source list
1941       pkgIndexFile *Index;
1942       if (Sources->FindIndex(Vf.File(),Index) == false)
1943             continue;
1944
1945       // only try to get a trusted package from another source if that source
1946       // is also trusted
1947       if(Trusted && !Index->IsTrusted())
1948          continue;
1949
1950       // Grab the text package record
1951       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1952       if (_error->PendingError() == true)
1953          return false;
1954
1955       string PkgFile = Parse.FileName();
1956       if (ForceHash.empty() == false)
1957       {
1958          if(stringcasecmp(ForceHash, "sha512") == 0)
1959             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1960          else if(stringcasecmp(ForceHash, "sha256") == 0)
1961             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1962          else if (stringcasecmp(ForceHash, "sha1") == 0)
1963             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1964          else
1965             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1966       }
1967       else
1968       {
1969          string Hash;
1970          if ((Hash = Parse.SHA512Hash()).empty() == false)
1971             ExpectedHash = HashString("SHA512", Hash);
1972          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1973             ExpectedHash = HashString("SHA256", Hash);
1974          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1975             ExpectedHash = HashString("SHA1", Hash);
1976          else
1977             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1978       }
1979       if (PkgFile.empty() == true)
1980          return _error->Error(_("The package index files are corrupted. No Filename: "
1981                               "field for package %s."),
1982                               Version.ParentPkg().Name());
1983
1984       Desc.URI = Index->ArchiveURI(PkgFile);
1985       Desc.Description = Index->ArchiveInfo(Version);
1986       Desc.Owner = this;
1987       Desc.ShortDesc = Version.ParentPkg().Name();
1988
1989       // See if we already have the file. (Legacy filenames)
1990       FileSize = Version->Size;
1991       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1992       struct stat Buf;
1993       if (stat(FinalFile.c_str(),&Buf) == 0)
1994       {
1995          // Make sure the size matches
1996          if ((unsigned long long)Buf.st_size == Version->Size)
1997          {
1998             Complete = true;
1999             Local = true;
2000             Status = StatDone;
2001             StoreFilename = DestFile = FinalFile;
2002             return true;
2003          }
2004
2005          /* Hmm, we have a file and its size does not match, this means it is
2006             an old style mismatched arch */
2007          unlink(FinalFile.c_str());
2008       }
2009
2010       // Check it again using the new style output filenames
2011       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2012       if (stat(FinalFile.c_str(),&Buf) == 0)
2013       {
2014          // Make sure the size matches
2015          if ((unsigned long long)Buf.st_size == Version->Size)
2016          {
2017             Complete = true;
2018             Local = true;
2019             Status = StatDone;
2020             StoreFilename = DestFile = FinalFile;
2021             return true;
2022          }
2023
2024          /* Hmm, we have a file and its size does not match, this shouldn't
2025             happen.. */
2026          unlink(FinalFile.c_str());
2027       }
2028
2029       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2030
2031       // Check the destination file
2032       if (stat(DestFile.c_str(),&Buf) == 0)
2033       {
2034          // Hmm, the partial file is too big, erase it
2035          if ((unsigned long long)Buf.st_size > Version->Size)
2036             unlink(DestFile.c_str());
2037          else
2038             PartialSize = Buf.st_size;
2039       }
2040
2041       // Disables download of archives - useful if no real installation follows,
2042       // e.g. if we are just interested in proposed installation order
2043       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2044       {
2045          Complete = true;
2046          Local = true;
2047          Status = StatDone;
2048          StoreFilename = DestFile = FinalFile;
2049          return true;
2050       }
2051
2052       // Create the item
2053       Local = false;
2054       Desc.URI = Index->ArchiveURI(PkgFile);
2055       Desc.Description = Index->ArchiveInfo(Version);
2056       Desc.Owner = this;
2057       Desc.ShortDesc = Version.ParentPkg().Name();
2058       QueueURI(Desc);
2059
2060       ++Vf;
2061       return true;
2062    }
2063    return false;
2064 }
2065                                                                         /*}}}*/
2066 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2067 // ---------------------------------------------------------------------
2068 /* */
2069 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2070                          pkgAcquire::MethodConfig *Cfg)
2071 {
2072    Item::Done(Message,Size,CalcHash,Cfg);
2073
2074    // Check the size
2075    if (Size != Version->Size)
2076    {
2077       RenameOnError(SizeMismatch);
2078       return;
2079    }
2080
2081    // Check the hash
2082    if(ExpectedHash.toStr() != CalcHash)
2083    {
2084       RenameOnError(HashSumMismatch);
2085       return;
2086    }
2087
2088    // Grab the output filename
2089    string FileName = LookupTag(Message,"Filename");
2090    if (FileName.empty() == true)
2091    {
2092       Status = StatError;
2093       ErrorText = "Method gave a blank filename";
2094       return;
2095    }
2096
2097    Complete = true;
2098
2099    // Reference filename
2100    if (FileName != DestFile)
2101    {
2102       StoreFilename = DestFile = FileName;
2103       Local = true;
2104       return;
2105    }
2106
2107    // Done, move it into position
2108    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2109    FinalFile += flNotDir(StoreFilename);
2110    Rename(DestFile,FinalFile);
2111
2112    StoreFilename = DestFile = FinalFile;
2113    Complete = true;
2114 }
2115                                                                         /*}}}*/
2116 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2117 // ---------------------------------------------------------------------
2118 /* Here we try other sources */
2119 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2120 {
2121    ErrorText = LookupTag(Message,"Message");
2122
2123    /* We don't really want to retry on failed media swaps, this prevents
2124       that. An interesting observation is that permanent failures are not
2125       recorded. */
2126    if (Cnf->Removable == true &&
2127        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2128    {
2129       // Vf = Version.FileList();
2130       while (Vf.end() == false) ++Vf;
2131       StoreFilename = string();
2132       Item::Failed(Message,Cnf);
2133       return;
2134    }
2135
2136    if (QueueNext() == false)
2137    {
2138       // This is the retry counter
2139       if (Retries != 0 &&
2140           Cnf->LocalOnly == false &&
2141           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2142       {
2143          Retries--;
2144          Vf = Version.FileList();
2145          if (QueueNext() == true)
2146             return;
2147       }
2148
2149       StoreFilename = string();
2150       Item::Failed(Message,Cnf);
2151    }
2152 }
2153                                                                         /*}}}*/
2154 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2155 // ---------------------------------------------------------------------
2156 bool pkgAcqArchive::IsTrusted()
2157 {
2158    return Trusted;
2159 }
2160                                                                         /*}}}*/
2161 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2162 // ---------------------------------------------------------------------
2163 /* */
2164 void pkgAcqArchive::Finished()
2165 {
2166    if (Status == pkgAcquire::Item::StatDone &&
2167        Complete == true)
2168       return;
2169    StoreFilename = string();
2170 }
2171                                                                         /*}}}*/
2172 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2173 // ---------------------------------------------------------------------
2174 /* The file is added to the queue */
2175 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2176                        unsigned long long Size,string Dsc,string ShortDesc,
2177                        const string &DestDir, const string &DestFilename,
2178                        bool IsIndexFile) :
2179                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2180 {
2181    Retries = _config->FindI("Acquire::Retries",0);
2182
2183    if(!DestFilename.empty())
2184       DestFile = DestFilename;
2185    else if(!DestDir.empty())
2186       DestFile = DestDir + "/" + flNotDir(URI);
2187    else
2188       DestFile = flNotDir(URI);
2189
2190    // Create the item
2191    Desc.URI = URI;
2192    Desc.Description = Dsc;
2193    Desc.Owner = this;
2194
2195    // Set the short description to the archive component
2196    Desc.ShortDesc = ShortDesc;
2197
2198    // Get the transfer sizes
2199    FileSize = Size;
2200    struct stat Buf;
2201    if (stat(DestFile.c_str(),&Buf) == 0)
2202    {
2203       // Hmm, the partial file is too big, erase it
2204       if ((unsigned long long)Buf.st_size > Size)
2205          unlink(DestFile.c_str());
2206       else
2207          PartialSize = Buf.st_size;
2208    }
2209
2210    QueueURI(Desc);
2211 }
2212                                                                         /*}}}*/
2213 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2214 // ---------------------------------------------------------------------
2215 /* */
2216 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2217                       pkgAcquire::MethodConfig *Cnf)
2218 {
2219    Item::Done(Message,Size,CalcHash,Cnf);
2220
2221    // Check the hash
2222    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2223    {
2224       RenameOnError(HashSumMismatch);
2225       return;
2226    }
2227
2228    string FileName = LookupTag(Message,"Filename");
2229    if (FileName.empty() == true)
2230    {
2231       Status = StatError;
2232       ErrorText = "Method gave a blank filename";
2233       return;
2234    }
2235
2236    Complete = true;
2237
2238    // The files timestamp matches
2239    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2240       return;
2241
2242    // We have to copy it into place
2243    if (FileName != DestFile)
2244    {
2245       Local = true;
2246       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2247           Cnf->Removable == true)
2248       {
2249          Desc.URI = "copy:" + FileName;
2250          QueueURI(Desc);
2251          return;
2252       }
2253
2254       // Erase the file if it is a symlink so we can overwrite it
2255       struct stat St;
2256       if (lstat(DestFile.c_str(),&St) == 0)
2257       {
2258          if (S_ISLNK(St.st_mode) != 0)
2259             unlink(DestFile.c_str());
2260       }
2261
2262       // Symlink the file
2263       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2264       {
2265          ErrorText = "Link to " + DestFile + " failure ";
2266          Status = StatError;
2267          Complete = false;
2268       }
2269    }
2270 }
2271                                                                         /*}}}*/
2272 // AcqFile::Failed - Failure handler                                    /*{{{*/
2273 // ---------------------------------------------------------------------
2274 /* Here we try other sources */
2275 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2276 {
2277    ErrorText = LookupTag(Message,"Message");
2278
2279    // This is the retry counter
2280    if (Retries != 0 &&
2281        Cnf->LocalOnly == false &&
2282        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2283    {
2284       Retries--;
2285       QueueURI(Desc);
2286       return;
2287    }
2288
2289    Item::Failed(Message,Cnf);
2290 }
2291                                                                         /*}}}*/
2292 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2293 // ---------------------------------------------------------------------
2294 /* The only header we use is the last-modified header. */
2295 string pkgAcqFile::Custom600Headers()
2296 {
2297    if (IsIndexFile)
2298       return "\nIndex-File: true";
2299    return "";
2300 }
2301                                                                         /*}}}*/