apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::Item::Item - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  57                        PartialSize(0), Mode(0), ID(0), Complete(false),
  58                        Local(false), QueueCounter(0)
  59 {
  60    Owner->Add(this);
  61    Status = StatIdle;
  62 }
  63                                                                         /*}}}*/
  64 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 pkgAcquire::Item::~Item()
  68 {
  69    Owner->Remove(this);
  70 }
  71                                                                         /*}}}*/
  72 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  73 // ---------------------------------------------------------------------
  74 /* We return to an idle state if there are still other queues that could
  75    fetch this object */
  76 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  77 {
  78    Status = StatIdle;
  79    ErrorText = LookupTag(Message,"Message");
  80    UsedMirror =  LookupTag(Message,"UsedMirror");
  81    if (QueueCounter <= 1)
  82    {
  83       /* This indicates that the file is not available right now but might
  84          be sometime later. If we do a retry cycle then this should be
  85          retried [CDROMs] */
  86       if (Cnf->LocalOnly == true &&
  87           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  88       {
  89          Status = StatIdle;
  90          Dequeue();
  91          return;
  92       }
  93
  94       Status = StatError;
  95       Dequeue();
  96    }
  97
  98    // report mirror failure back to LP if we actually use a mirror
  99    string FailReason = LookupTag(Message, "FailReason");
 100    if(FailReason.size() != 0)
 101       ReportMirrorFailure(FailReason);
 102    else
 103       ReportMirrorFailure(ErrorText);
 104 }
 105                                                                         /*}}}*/
 106 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* Stash status and the file size. Note that setting Complete means
 109    sub-phases of the acquire process such as decompresion are operating */
 110 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 111 {
 112    Status = StatFetching;
 113    if (FileSize == 0 && Complete == false)
 114       FileSize = Size;
 115 }
 116                                                                         /*}}}*/
 117 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/,
 121                             pkgAcquire::MethodConfig * /*Cnf*/)
 122 {
 123    // We just downloaded something..
 124    string FileName = LookupTag(Message,"Filename");
 125    UsedMirror =  LookupTag(Message,"UsedMirror");
 126    if (Complete == false && !Local && FileName == DestFile)
 127    {
 128       if (Owner->Log != 0)
 129          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 130    }
 131
 132    if (FileSize == 0)
 133       FileSize= Size;
 134    Status = StatDone;
 135    ErrorText = string();
 136    Owner->Dequeue(this);
 137 }
 138                                                                         /*}}}*/
 139 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This helper function is used by a lot of item methods as their final
 142    step */
 143 void pkgAcquire::Item::Rename(string From,string To)
 144 {
 145    if (rename(From.c_str(),To.c_str()) != 0)
 146    {
 147       char S[300];
 148       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 149               From.c_str(),To.c_str());
 150       Status = StatError;
 151       ErrorText = S;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 156 {
 157    if(FileExists(DestFile))
 158       Rename(DestFile, DestFile + ".FAILED");
 159
 160    switch (error)
 161    {
 162       case HashSumMismatch:
 163          ErrorText = _("Hash Sum mismatch");
 164          Status = StatAuthError;
 165          ReportMirrorFailure("HashChecksumFailure");
 166          break;
 167       case SizeMismatch:
 168          ErrorText = _("Size mismatch");
 169          Status = StatAuthError;
 170          ReportMirrorFailure("SizeFailure");
 171          break;
 172       case InvalidFormat:
 173          ErrorText = _("Invalid file format");
 174          Status = StatError;
 175          // do not report as usually its not the mirrors fault, but Portal/Proxy
 176          break;
 177    }
 178    return false;
 179 }
 180                                                                         /*}}}*/
 181 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 182 // ---------------------------------------------------------------------
 183 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 184 {
 185    // we only act if a mirror was used at all
 186    if(UsedMirror.empty())
 187       return;
 188 #if 0
 189    std::cerr << "\nReportMirrorFailure: "
 190              << UsedMirror
 191              << " Uri: " << DescURI()
 192              << " FailCode: "
 193              << FailCode << std::endl;
 194 #endif
 195    const char *Args[40];
 196    unsigned int i = 0;
 197    string report = _config->Find("Methods::Mirror::ProblemReporting",
 198                                  "/usr/lib/apt/apt-report-mirror-failure");
 199    if(!FileExists(report))
 200       return;
 201    Args[i++] = report.c_str();
 202    Args[i++] = UsedMirror.c_str();
 203    Args[i++] = DescURI().c_str();
 204    Args[i++] = FailCode.c_str();
 205    Args[i++] = NULL;
 206    pid_t pid = ExecFork();
 207    if(pid < 0)
 208    {
 209       _error->Error("ReportMirrorFailure Fork failed");
 210       return;
 211    }
 212    else if(pid == 0)
 213    {
 214       execvp(Args[0], (char**)Args);
 215       std::cerr << "Could not exec " << Args[0] << std::endl;
 216       _exit(100);
 217    }
 218    if(!ExecWait(pid, "report-mirror-failure"))
 219    {
 220       _error->Warning("Couldn't report problem to '%s'",
 221                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 222    }
 223 }
 224                                                                         /*}}}*/
 225 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 226 // ---------------------------------------------------------------------
 227 /* Get a sub-index file based on checksums from a 'master' file and
 228    possibly query additional files */
 229 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 230                                  string const &URIDesc, string const &ShortDesc,
 231                                  HashString const &ExpectedHash)
 232    : Item(Owner), ExpectedHash(ExpectedHash)
 233 {
 234    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 235    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 236
 237    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 238    DestFile += URItoFileName(URI);
 239
 240    Desc.URI = URI;
 241    Desc.Description = URIDesc;
 242    Desc.Owner = this;
 243    Desc.ShortDesc = ShortDesc;
 244
 245    QueueURI(Desc);
 246
 247    if(Debug)
 248       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 249 }
 250                                                                         /*}}}*/
 251 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 252 // ---------------------------------------------------------------------
 253 /* The only header we use is the last-modified header. */
 254 string pkgAcqSubIndex::Custom600Headers()
 255 {
 256    string Final = _config->FindDir("Dir::State::lists");
 257    Final += URItoFileName(Desc.URI);
 258
 259    struct stat Buf;
 260    if (stat(Final.c_str(),&Buf) != 0)
 261       return "\nIndex-File: true\nFail-Ignore: true\n";
 262    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 263 }
 264                                                                         /*}}}*/
 265 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 266 {
 267    if(Debug)
 268       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 269
 270    Complete = false;
 271    Status = StatDone;
 272    Dequeue();
 273
 274    // No good Index is provided
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 278                            pkgAcquire::MethodConfig *Cnf)
 279 {
 280    if(Debug)
 281       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 282
 283    string FileName = LookupTag(Message,"Filename");
 284    if (FileName.empty() == true)
 285    {
 286       Status = StatError;
 287       ErrorText = "Method gave a blank filename";
 288       return;
 289    }
 290
 291    if (FileName != DestFile)
 292    {
 293       Local = true;
 294       Desc.URI = "copy:" + FileName;
 295       QueueURI(Desc);
 296       return;
 297    }
 298
 299    Item::Done(Message,Size,Md5Hash,Cnf);
 300
 301    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 302
 303    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 304    indexRecords SubIndexParser;
 305    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 306       Status = StatError;
 307       ErrorText = SubIndexParser.ErrorText;
 308       return;
 309    }
 310
 311    // success in downloading the index
 312    // rename the index
 313    if(Debug)
 314       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 315    Rename(DestFile,FinalFile);
 316    chmod(FinalFile.c_str(),0644);
 317    DestFile = FinalFile;
 318
 319    if(ParseIndex(DestFile) == false)
 320       return Failed("", NULL);
 321
 322    Complete = true;
 323    Status = StatDone;
 324    Dequeue();
 325    return;
 326 }
 327                                                                         /*}}}*/
 328 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 329 {
 330    indexRecords SubIndexParser;
 331    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 332       return false;
 333    // so something with the downloaded index
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* Get the DiffIndex file first and see if there are patches available
 340  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 341  * patches. If anything goes wrong in that process, it will fall back to
 342  * the original packages file
 343  */
 344 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 345                                  string URI,string URIDesc,string ShortDesc,
 346                                  HashString ExpectedHash)
 347    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 348      Description(URIDesc)
 349 {
 350
 351    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 352
 353    Desc.Description = URIDesc + "/DiffIndex";
 354    Desc.Owner = this;
 355    Desc.ShortDesc = ShortDesc;
 356    Desc.URI = URI + ".diff/Index";
 357
 358    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 359    DestFile += URItoFileName(URI) + string(".DiffIndex");
 360
 361    if(Debug)
 362       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 363
 364    // look for the current package file
 365    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 366    CurrentPackagesFile += URItoFileName(RealURI);
 367
 368    // FIXME: this file:/ check is a hack to prevent fetching
 369    //        from local sources. this is really silly, and
 370    //        should be fixed cleanly as soon as possible
 371    if(!FileExists(CurrentPackagesFile) ||
 372       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 373    {
 374       // we don't have a pkg file or we don't want to queue
 375       if(Debug)
 376          std::clog << "No index file, local or canceld by user" << std::endl;
 377       Failed("", NULL);
 378       return;
 379    }
 380
 381    if(Debug)
 382       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 383          << CurrentPackagesFile << std::endl;
 384
 385    QueueURI(Desc);
 386
 387 }
 388                                                                         /*}}}*/
 389 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* The only header we use is the last-modified header. */
 392 string pkgAcqDiffIndex::Custom600Headers()
 393 {
 394    string Final = _config->FindDir("Dir::State::lists");
 395    Final += URItoFileName(RealURI) + string(".IndexDiff");
 396
 397    if(Debug)
 398       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 399
 400    struct stat Buf;
 401    if (stat(Final.c_str(),&Buf) != 0)
 402       return "\nIndex-File: true";
 403
 404    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 405 }
 406                                                                         /*}}}*/
 407 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 408 {
 409    if(Debug)
 410       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 411          << std::endl;
 412
 413    pkgTagSection Tags;
 414    string ServerSha1;
 415    vector<DiffInfo> available_patches;
 416
 417    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 418    pkgTagFile TF(&Fd);
 419    if (_error->PendingError() == true)
 420       return false;
 421
 422    if(TF.Step(Tags) == true)
 423    {
 424       bool found = false;
 425       DiffInfo d;
 426       string size;
 427
 428       string const tmp = Tags.FindS("SHA1-Current");
 429       std::stringstream ss(tmp);
 430       ss >> ServerSha1 >> size;
 431       unsigned long const ServerSize = atol(size.c_str());
 432
 433       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 434       SHA1Summation SHA1;
 435       SHA1.AddFD(fd);
 436       string const local_sha1 = SHA1.Result();
 437
 438       if(local_sha1 == ServerSha1)
 439       {
 440          // we have the same sha1 as the server so we are done here
 441          if(Debug)
 442             std::clog << "Package file is up-to-date" << std::endl;
 443          // list cleanup needs to know that this file as well as the already
 444          // present index is ours, so we create an empty diff to save it for us
 445          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 446                ExpectedHash, ServerSha1, available_patches);
 447          return true;
 448       }
 449       else
 450       {
 451          if(Debug)
 452             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 453
 454          // check the historie and see what patches we need
 455          string const history = Tags.FindS("SHA1-History");
 456          std::stringstream hist(history);
 457          while(hist >> d.sha1 >> size >> d.file)
 458          {
 459             // read until the first match is found
 460             // from that point on, we probably need all diffs
 461             if(d.sha1 == local_sha1)
 462                found=true;
 463             else if (found == false)
 464                continue;
 465
 466             if(Debug)
 467                std::clog << "Need to get diff: " << d.file << std::endl;
 468             available_patches.push_back(d);
 469          }
 470
 471          if (available_patches.empty() == false)
 472          {
 473             // patching with too many files is rather slow compared to a fast download
 474             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 475             if (fileLimit != 0 && fileLimit < available_patches.size())
 476             {
 477                if (Debug)
 478                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 479                         << ") so fallback to complete download" << std::endl;
 480                return false;
 481             }
 482
 483             // see if the patches are too big
 484             found = false; // it was true and it will be true again at the end
 485             d = *available_patches.begin();
 486             string const firstPatch = d.file;
 487             unsigned long patchesSize = 0;
 488             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 489             while(patches >> d.sha1 >> size >> d.file)
 490             {
 491                if (firstPatch == d.file)
 492                   found = true;
 493                else if (found == false)
 494                   continue;
 495
 496                patchesSize += atol(size.c_str());
 497             }
 498             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 499             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 500             {
 501                if (Debug)
 502                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 503                         << ") so fallback to complete download" << std::endl;
 504                return false;
 505             }
 506          }
 507       }
 508
 509       // we have something, queue the next diff
 510       if(found)
 511       {
 512          // queue the diffs
 513          string::size_type const last_space = Description.rfind(" ");
 514          if(last_space != string::npos)
 515             Description.erase(last_space, Description.size()-last_space);
 516
 517          /* decide if we should download patches one by one or in one go:
 518             The first is good if the server merges patches, but many don't so client
 519             based merging can be attempt in which case the second is better.
 520             "bad things" will happen if patches are merged on the server,
 521             but client side merging is attempt as well */
 522          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 523          if (pdiff_merge == true)
 524          {
 525             // reprepro adds this flag if it has merged patches on the server
 526             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 527             pdiff_merge = (precedence != "merged");
 528          }
 529
 530          if (pdiff_merge == false)
 531             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 532                   ExpectedHash, ServerSha1, available_patches);
 533          else
 534          {
 535             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 536             for(size_t i = 0; i < available_patches.size(); ++i)
 537                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 538                      available_patches[i], diffs);
 539          }
 540
 541          Complete = false;
 542          Status = StatDone;
 543          Dequeue();
 544          return true;
 545       }
 546    }
 547
 548    // Nothing found, report and return false
 549    // Failing here is ok, if we return false later, the full
 550    // IndexFile is queued
 551    if(Debug)
 552       std::clog << "Can't find a patch in the index file" << std::endl;
 553    return false;
 554 }
 555                                                                         /*}}}*/
 556 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 557 {
 558    if(Debug)
 559       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 560                 << "Falling back to normal index file acquire" << std::endl;
 561
 562    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 563                    ExpectedHash);
 564
 565    Complete = false;
 566    Status = StatDone;
 567    Dequeue();
 568 }
 569                                                                         /*}}}*/
 570 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 571                            pkgAcquire::MethodConfig *Cnf)
 572 {
 573    if(Debug)
 574       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 575
 576    Item::Done(Message,Size,Md5Hash,Cnf);
 577
 578    string FinalFile;
 579    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 580
 581    // success in downloading the index
 582    // rename the index
 583    FinalFile += string(".IndexDiff");
 584    if(Debug)
 585       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 586                 << std::endl;
 587    Rename(DestFile,FinalFile);
 588    chmod(FinalFile.c_str(),0644);
 589    DestFile = FinalFile;
 590
 591    if(!ParseDiffIndex(DestFile))
 592       return Failed("", NULL);
 593
 594    Complete = true;
 595    Status = StatDone;
 596    Dequeue();
 597    return;
 598 }
 599                                                                         /*}}}*/
 600 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* The package diff is added to the queue. one object is constructed
 603  * for each diff and the index
 604  */
 605 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 606                                    string URI,string URIDesc,string ShortDesc,
 607                                    HashString ExpectedHash,
 608                                    string ServerSha1,
 609                                    vector<DiffInfo> diffs)
 610    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 611      available_patches(diffs), ServerSha1(ServerSha1)
 612 {
 613
 614    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 615    DestFile += URItoFileName(URI);
 616
 617    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 618
 619    Description = URIDesc;
 620    Desc.Owner = this;
 621    Desc.ShortDesc = ShortDesc;
 622
 623    if(available_patches.empty() == true)
 624    {
 625       // we are done (yeah!)
 626       Finish(true);
 627    }
 628    else
 629    {
 630       // get the next diff
 631       State = StateFetchDiff;
 632       QueueNextDiff();
 633    }
 634 }
 635                                                                         /*}}}*/
 636 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 637 {
 638    if(Debug)
 639       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 640                 << "Falling back to normal index file acquire" << std::endl;
 641    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 642                    ExpectedHash);
 643    Finish();
 644 }
 645                                                                         /*}}}*/
 646 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 647 void pkgAcqIndexDiffs::Finish(bool allDone)
 648 {
 649    // we restore the original name, this is required, otherwise
 650    // the file will be cleaned
 651    if(allDone)
 652    {
 653       DestFile = _config->FindDir("Dir::State::lists");
 654       DestFile += URItoFileName(RealURI);
 655
 656       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 657       {
 658          RenameOnError(HashSumMismatch);
 659          Dequeue();
 660          return;
 661       }
 662
 663       // this is for the "real" finish
 664       Complete = true;
 665       Status = StatDone;
 666       Dequeue();
 667       if(Debug)
 668          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 669       return;
 670    }
 671
 672    if(Debug)
 673       std::clog << "Finishing: " << Desc.URI << std::endl;
 674    Complete = false;
 675    Status = StatDone;
 676    Dequeue();
 677    return;
 678 }
 679                                                                         /*}}}*/
 680 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 681 {
 682
 683    // calc sha1 of the just patched file
 684    string FinalFile = _config->FindDir("Dir::State::lists");
 685    FinalFile += URItoFileName(RealURI);
 686
 687    FileFd fd(FinalFile, FileFd::ReadOnly);
 688    SHA1Summation SHA1;
 689    SHA1.AddFD(fd);
 690    string local_sha1 = string(SHA1.Result());
 691    if(Debug)
 692       std::clog << "QueueNextDiff: "
 693                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 694
 695    // final file reached before all patches are applied
 696    if(local_sha1 == ServerSha1)
 697    {
 698       Finish(true);
 699       return true;
 700    }
 701
 702    // remove all patches until the next matching patch is found
 703    // this requires the Index file to be ordered
 704    for(vector<DiffInfo>::iterator I=available_patches.begin();
 705        available_patches.empty() == false &&
 706           I != available_patches.end() &&
 707           I->sha1 != local_sha1;
 708        ++I)
 709    {
 710       available_patches.erase(I);
 711    }
 712
 713    // error checking and falling back if no patch was found
 714    if(available_patches.empty() == true)
 715    {
 716       Failed("", NULL);
 717       return false;
 718    }
 719
 720    // queue the right diff
 721    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 722    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 723    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 724    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 725
 726    if(Debug)
 727       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 728
 729    QueueURI(Desc);
 730
 731    return true;
 732 }
 733                                                                         /*}}}*/
 734 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 735                             pkgAcquire::MethodConfig *Cnf)
 736 {
 737    if(Debug)
 738       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 739
 740    Item::Done(Message,Size,Md5Hash,Cnf);
 741
 742    string FinalFile;
 743    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 744
 745    // success in downloading a diff, enter ApplyDiff state
 746    if(State == StateFetchDiff)
 747    {
 748
 749       // rred excepts the patch as $FinalFile.ed
 750       Rename(DestFile,FinalFile+".ed");
 751
 752       if(Debug)
 753          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 754
 755       State = StateApplyDiff;
 756       Local = true;
 757       Desc.URI = "rred:" + FinalFile;
 758       QueueURI(Desc);
 759       Mode = "rred";
 760       return;
 761    }
 762
 763
 764    // success in download/apply a diff, queue next (if needed)
 765    if(State == StateApplyDiff)
 766    {
 767       // remove the just applied patch
 768       available_patches.erase(available_patches.begin());
 769       unlink((FinalFile + ".ed").c_str());
 770
 771       // move into place
 772       if(Debug)
 773       {
 774          std::clog << "Moving patched file in place: " << std::endl
 775                    << DestFile << " -> " << FinalFile << std::endl;
 776       }
 777       Rename(DestFile,FinalFile);
 778       chmod(FinalFile.c_str(),0644);
 779
 780       // see if there is more to download
 781       if(available_patches.empty() == false) {
 782          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 783                               ExpectedHash, ServerSha1, available_patches);
 784          return Finish();
 785       } else
 786          return Finish(true);
 787    }
 788 }
 789                                                                         /*}}}*/
 790 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 791 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 792                                    string const &URI, string const &URIDesc,
 793                                    string const &ShortDesc, HashString const &ExpectedHash,
 794                                    DiffInfo const &patch,
 795                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 796    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 797      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 798 {
 799
 800    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 801    DestFile += URItoFileName(URI);
 802
 803    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 804
 805    Description = URIDesc;
 806    Desc.Owner = this;
 807    Desc.ShortDesc = ShortDesc;
 808
 809    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 810    Desc.Description = Description + " " + patch.file + string(".pdiff");
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 813
 814    if(Debug)
 815       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 816
 817    QueueURI(Desc);
 818 }
 819                                                                         /*}}}*/
 820 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 821 {
 822    if(Debug)
 823       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 824    Complete = false;
 825    Status = StatDone;
 826    Dequeue();
 827
 828    // check if we are the first to fail, otherwise we are done here
 829    State = StateDoneDiff;
 830    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 831          I != allPatches->end(); ++I)
 832       if ((*I)->State == StateErrorDiff)
 833          return;
 834
 835    // first failure means we should fallback
 836    State = StateErrorDiff;
 837    std::clog << "Falling back to normal index file acquire" << std::endl;
 838    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 839                    ExpectedHash);
 840 }
 841                                                                         /*}}}*/
 842 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 843                             pkgAcquire::MethodConfig *Cnf)
 844 {
 845    if(Debug)
 846       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 847
 848    Item::Done(Message,Size,Md5Hash,Cnf);
 849
 850    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 851
 852    if (State == StateFetchDiff)
 853    {
 854       // rred expects the patch as $FinalFile.ed.$patchname.gz
 855       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 856
 857       // check if this is the last completed diff
 858       State = StateDoneDiff;
 859       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 860             I != allPatches->end(); ++I)
 861          if ((*I)->State != StateDoneDiff)
 862          {
 863             if(Debug)
 864                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 865             return;
 866          }
 867
 868       // this is the last completed diff, so we are ready to apply now
 869       State = StateApplyDiff;
 870
 871       if(Debug)
 872          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 873
 874       Local = true;
 875       Desc.URI = "rred:" + FinalFile;
 876       QueueURI(Desc);
 877       Mode = "rred";
 878       return;
 879    }
 880    // success in download/apply all diffs, clean up
 881    else if (State == StateApplyDiff)
 882    {
 883       // see if we really got the expected file
 884       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 885       {
 886          RenameOnError(HashSumMismatch);
 887          return;
 888       }
 889
 890       // move the result into place
 891       if(Debug)
 892          std::clog << "Moving patched file in place: " << std::endl
 893                    << DestFile << " -> " << FinalFile << std::endl;
 894       Rename(DestFile, FinalFile);
 895       chmod(FinalFile.c_str(), 0644);
 896
 897       // otherwise lists cleanup will eat the file
 898       DestFile = FinalFile;
 899
 900       // ensure the ed's are gone regardless of list-cleanup
 901       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 902             I != allPatches->end(); ++I)
 903       {
 904             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 905             unlink(patch.c_str());
 906       }
 907
 908       // all set and done
 909       Complete = true;
 910       if(Debug)
 911          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 912    }
 913 }
 914                                                                         /*}}}*/
 915 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* The package file is added to the queue and a second class is
 918    instantiated to fetch the revision file */
 919 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 920                          string URI,string URIDesc,string ShortDesc,
 921                          HashString ExpectedHash, string comprExt)
 922    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 923 {
 924    if(comprExt.empty() == true)
 925    {
 926       // autoselect the compression method
 927       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 928       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 929          comprExt.append(*t).append(" ");
 930       if (comprExt.empty() == false)
 931          comprExt.erase(comprExt.end()-1);
 932    }
 933    CompressionExtension = comprExt;
 934
 935    Init(URI, URIDesc, ShortDesc);
 936 }
 937 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 938                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 939    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 940 {
 941    // autoselect the compression method
 942    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 943    CompressionExtension = "";
 944    if (ExpectedHash.empty() == false)
 945    {
 946       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 947          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 948             CompressionExtension.append(*t).append(" ");
 949    }
 950    else
 951    {
 952       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 953          CompressionExtension.append(*t).append(" ");
 954    }
 955    if (CompressionExtension.empty() == false)
 956       CompressionExtension.erase(CompressionExtension.end()-1);
 957
 958    Init(Target->URI, Target->Description, Target->ShortDesc);
 959 }
 960                                                                         /*}}}*/
 961 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 962 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 963    Decompression = false;
 964    Erase = false;
 965
 966    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 967    DestFile += URItoFileName(URI);
 968
 969    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 970    if (comprExt == "uncompressed")
 971       Desc.URI = URI;
 972    else
 973       Desc.URI = URI + '.' + comprExt;
 974
 975    Desc.Description = URIDesc;
 976    Desc.Owner = this;
 977    Desc.ShortDesc = ShortDesc;
 978
 979    QueueURI(Desc);
 980 }
 981                                                                         /*}}}*/
 982 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 983 // ---------------------------------------------------------------------
 984 /* The only header we use is the last-modified header. */
 985 string pkgAcqIndex::Custom600Headers()
 986 {
 987    string Final = _config->FindDir("Dir::State::lists");
 988    Final += URItoFileName(RealURI);
 989    if (_config->FindB("Acquire::GzipIndexes",false))
 990       Final += ".gz";
 991
 992    string msg = "\nIndex-File: true";
 993    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 994    //        seems to be difficult without breaking ABI
 995    if (ShortDesc().find("Translation") != 0)
 996       msg += "\nFail-Ignore: true";
 997    struct stat Buf;
 998    if (stat(Final.c_str(),&Buf) == 0)
 999       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1000
1001    return msg;
1002 }
1003                                                                         /*}}}*/
1004 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1005 {
1006    size_t const nextExt = CompressionExtension.find(' ');
1007    if (nextExt != std::string::npos)
1008    {
1009       CompressionExtension = CompressionExtension.substr(nextExt+1);
1010       Init(RealURI, Desc.Description, Desc.ShortDesc);
1011       return;
1012    }
1013
1014    // on decompression failure, remove bad versions in partial/
1015    if (Decompression && Erase) {
1016       string s = _config->FindDir("Dir::State::lists") + "partial/";
1017       s.append(URItoFileName(RealURI));
1018       unlink(s.c_str());
1019    }
1020
1021    Item::Failed(Message,Cnf);
1022 }
1023                                                                         /*}}}*/
1024 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1025 // ---------------------------------------------------------------------
1026 /* This goes through a number of states.. On the initial fetch the
1027    method could possibly return an alternate filename which points
1028    to the uncompressed version of the file. If this is so the file
1029    is copied into the partial directory. In all other cases the file
1030    is decompressed with a gzip uri. */
1031 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1032                        pkgAcquire::MethodConfig *Cfg)
1033 {
1034    Item::Done(Message,Size,Hash,Cfg);
1035
1036    if (Decompression == true)
1037    {
1038       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1039       {
1040          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1041          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1042       }
1043
1044       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1045       {
1046          RenameOnError(HashSumMismatch);
1047          return;
1048       }
1049
1050       // FIXME: this can go away once we only ever download stuff that
1051       //        has a valid hash and we never do GET based probing
1052       //
1053       /* Always verify the index file for correctness (all indexes must
1054        * have a Package field) (LP: #346386) (Closes: #627642)
1055        */
1056       FileFd fd(DestFile, FileFd::ReadOnly);
1057       // Only test for correctness if the file is not empty (empty is ok)
1058       if (fd.FileSize() > 0)
1059       {
1060          pkgTagSection sec;
1061          pkgTagFile tag(&fd);
1062
1063          // all our current indexes have a field 'Package' in each section
1064          if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1065          {
1066             RenameOnError(InvalidFormat);
1067             return;
1068          }
1069       }
1070
1071       // Done, move it into position
1072       string FinalFile = _config->FindDir("Dir::State::lists");
1073       FinalFile += URItoFileName(RealURI);
1074       Rename(DestFile,FinalFile);
1075       chmod(FinalFile.c_str(),0644);
1076
1077       /* We restore the original name to DestFile so that the clean operation
1078          will work OK */
1079       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1080       DestFile += URItoFileName(RealURI);
1081
1082       // Remove the compressed version.
1083       if (Erase == true)
1084          unlink(DestFile.c_str());
1085       return;
1086    }
1087
1088    Erase = false;
1089    Complete = true;
1090
1091    // Handle the unzipd case
1092    string FileName = LookupTag(Message,"Alt-Filename");
1093    if (FileName.empty() == false)
1094    {
1095       // The files timestamp matches
1096       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1097          return;
1098       Decompression = true;
1099       Local = true;
1100       DestFile += ".decomp";
1101       Desc.URI = "copy:" + FileName;
1102       QueueURI(Desc);
1103       Mode = "copy";
1104       return;
1105    }
1106
1107    FileName = LookupTag(Message,"Filename");
1108    if (FileName.empty() == true)
1109    {
1110       Status = StatError;
1111       ErrorText = "Method gave a blank filename";
1112    }
1113
1114    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1115
1116    // The files timestamp matches
1117    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1118        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1119           // Update DestFile for .gz suffix so that the clean operation keeps it
1120           DestFile += ".gz";
1121       return;
1122     }
1123
1124    if (FileName == DestFile)
1125       Erase = true;
1126    else
1127       Local = true;
1128
1129    string decompProg;
1130
1131    // If we enable compressed indexes and already have gzip, keep it
1132    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1133       string FinalFile = _config->FindDir("Dir::State::lists");
1134       FinalFile += URItoFileName(RealURI) + ".gz";
1135       Rename(DestFile,FinalFile);
1136       chmod(FinalFile.c_str(),0644);
1137
1138       // Update DestFile for .gz suffix so that the clean operation keeps it
1139       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1140       DestFile += URItoFileName(RealURI) + ".gz";
1141       return;
1142     }
1143
1144    // get the binary name for your used compression type
1145    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1146    if(decompProg.empty() == false);
1147    else if(compExt == "uncompressed")
1148       decompProg = "copy";
1149    else {
1150       _error->Error("Unsupported extension: %s", compExt.c_str());
1151       return;
1152    }
1153
1154    Decompression = true;
1155    DestFile += ".decomp";
1156    Desc.URI = decompProg + ":" + FileName;
1157    QueueURI(Desc);
1158
1159    // FIXME: this points to a c++ string that goes out of scope
1160    Mode = decompProg.c_str();
1161 }
1162                                                                         /*}}}*/
1163 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1164 // ---------------------------------------------------------------------
1165 /* The Translation file is added to the queue */
1166 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1167                             string URI,string URIDesc,string ShortDesc)
1168   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1169 {
1170 }
1171 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1172                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1173   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1174 {
1175 }
1176                                                                         /*}}}*/
1177 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1178 // ---------------------------------------------------------------------
1179 string pkgAcqIndexTrans::Custom600Headers()
1180 {
1181    string Final = _config->FindDir("Dir::State::lists");
1182    Final += URItoFileName(RealURI);
1183
1184    struct stat Buf;
1185    if (stat(Final.c_str(),&Buf) != 0)
1186       return "\nFail-Ignore: true\nIndex-File: true";
1187    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1188 }
1189                                                                         /*}}}*/
1190 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1191 // ---------------------------------------------------------------------
1192 /* */
1193 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1194 {
1195    size_t const nextExt = CompressionExtension.find(' ');
1196    if (nextExt != std::string::npos)
1197    {
1198       CompressionExtension = CompressionExtension.substr(nextExt+1);
1199       Init(RealURI, Desc.Description, Desc.ShortDesc);
1200       Status = StatIdle;
1201       return;
1202    }
1203
1204    if (Cnf->LocalOnly == true ||
1205        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1206    {
1207       // Ignore this
1208       Status = StatDone;
1209       Complete = false;
1210       Dequeue();
1211       return;
1212    }
1213
1214    Item::Failed(Message,Cnf);
1215 }
1216                                                                         /*}}}*/
1217 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1218                              string URI,string URIDesc,string ShortDesc,
1219                              string MetaIndexURI, string MetaIndexURIDesc,
1220                              string MetaIndexShortDesc,
1221                              const vector<IndexTarget*>* IndexTargets,
1222                              indexRecords* MetaIndexParser) :
1223    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1224    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1225    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1226 {
1227    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1228    DestFile += URItoFileName(URI);
1229
1230    // remove any partial downloaded sig-file in partial/.
1231    // it may confuse proxies and is too small to warrant a
1232    // partial download anyway
1233    unlink(DestFile.c_str());
1234
1235    // Create the item
1236    Desc.Description = URIDesc;
1237    Desc.Owner = this;
1238    Desc.ShortDesc = ShortDesc;
1239    Desc.URI = URI;
1240
1241    string Final = _config->FindDir("Dir::State::lists");
1242    Final += URItoFileName(RealURI);
1243    if (RealFileExists(Final) == true)
1244    {
1245       // File was already in place.  It needs to be re-downloaded/verified
1246       // because Release might have changed, we do give it a different
1247       // name than DestFile because otherwise the http method will
1248       // send If-Range requests and there are too many broken servers
1249       // out there that do not understand them
1250       LastGoodSig = DestFile+".reverify";
1251       Rename(Final,LastGoodSig);
1252    }
1253
1254    QueueURI(Desc);
1255 }
1256                                                                         /*}}}*/
1257 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1258 {
1259    // if the file was never queued undo file-changes done in the constructor
1260    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1261          LastGoodSig.empty() == false)
1262    {
1263       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1264       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1265          Rename(LastGoodSig, Final);
1266    }
1267
1268 }
1269                                                                         /*}}}*/
1270 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1271 // ---------------------------------------------------------------------
1272 /* The only header we use is the last-modified header. */
1273 string pkgAcqMetaSig::Custom600Headers()
1274 {
1275    struct stat Buf;
1276    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1277       return "\nIndex-File: true";
1278
1279    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1280 }
1281
1282 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1283                          pkgAcquire::MethodConfig *Cfg)
1284 {
1285    Item::Done(Message,Size,MD5,Cfg);
1286
1287    string FileName = LookupTag(Message,"Filename");
1288    if (FileName.empty() == true)
1289    {
1290       Status = StatError;
1291       ErrorText = "Method gave a blank filename";
1292       return;
1293    }
1294
1295    if (FileName != DestFile)
1296    {
1297       // We have to copy it into place
1298       Local = true;
1299       Desc.URI = "copy:" + FileName;
1300       QueueURI(Desc);
1301       return;
1302    }
1303
1304    Complete = true;
1305
1306    // put the last known good file back on i-m-s hit (it will
1307    // be re-verified again)
1308    // Else do nothing, we have the new file in DestFile then
1309    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1310       Rename(LastGoodSig, DestFile);
1311
1312    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1313    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1314                        MetaIndexShortDesc,  DestFile, IndexTargets,
1315                        MetaIndexParser);
1316
1317 }
1318                                                                         /*}}}*/
1319 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1320 {
1321    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1322
1323    // if we get a network error we fail gracefully
1324    if(Status == StatTransientNetworkError)
1325    {
1326       Item::Failed(Message,Cnf);
1327       // move the sigfile back on transient network failures
1328       if(FileExists(LastGoodSig))
1329          Rename(LastGoodSig,Final);
1330
1331       // set the status back to , Item::Failed likes to reset it
1332       Status = pkgAcquire::Item::StatTransientNetworkError;
1333       return;
1334    }
1335
1336    // Delete any existing sigfile when the acquire failed
1337    unlink(Final.c_str());
1338
1339    // queue a pkgAcqMetaIndex with no sigfile
1340    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1341                        "", IndexTargets, MetaIndexParser);
1342
1343    if (Cnf->LocalOnly == true ||
1344        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1345    {
1346       // Ignore this
1347       Status = StatDone;
1348       Complete = false;
1349       Dequeue();
1350       return;
1351    }
1352
1353    Item::Failed(Message,Cnf);
1354 }
1355                                                                         /*}}}*/
1356 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1357                                  string URI,string URIDesc,string ShortDesc,
1358                                  string SigFile,
1359                                  const vector<struct IndexTarget*>* IndexTargets,
1360                                  indexRecords* MetaIndexParser) :
1361    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1362    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1363 {
1364    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1365    DestFile += URItoFileName(URI);
1366
1367    // Create the item
1368    Desc.Description = URIDesc;
1369    Desc.Owner = this;
1370    Desc.ShortDesc = ShortDesc;
1371    Desc.URI = URI;
1372
1373    QueueURI(Desc);
1374 }
1375                                                                         /*}}}*/
1376 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1377 // ---------------------------------------------------------------------
1378 /* The only header we use is the last-modified header. */
1379 string pkgAcqMetaIndex::Custom600Headers()
1380 {
1381    string Final = _config->FindDir("Dir::State::lists");
1382    Final += URItoFileName(RealURI);
1383
1384    struct stat Buf;
1385    if (stat(Final.c_str(),&Buf) != 0)
1386       return "\nIndex-File: true";
1387
1388    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1389 }
1390                                                                         /*}}}*/
1391 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1392                            pkgAcquire::MethodConfig *Cfg)
1393 {
1394    Item::Done(Message,Size,Hash,Cfg);
1395
1396    // MetaIndexes are done in two passes: one to download the
1397    // metaindex with an appropriate method, and a second to verify it
1398    // with the gpgv method
1399
1400    if (AuthPass == true)
1401    {
1402       AuthDone(Message);
1403
1404       // all cool, move Release file into place
1405       Complete = true;
1406    }
1407    else
1408    {
1409       RetrievalDone(Message);
1410       if (!Complete)
1411          // Still more retrieving to do
1412          return;
1413
1414       if (SigFile == "")
1415       {
1416          // There was no signature file, so we are finished.  Download
1417          // the indexes and do only hashsum verification if possible
1418          MetaIndexParser->Load(DestFile);
1419          QueueIndexes(false);
1420       }
1421       else
1422       {
1423          // There was a signature file, so pass it to gpgv for
1424          // verification
1425
1426          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1427             std::cerr << "Metaindex acquired, queueing gpg verification ("
1428                       << SigFile << "," << DestFile << ")\n";
1429          AuthPass = true;
1430          Desc.URI = "gpgv:" + SigFile;
1431          QueueURI(Desc);
1432          Mode = "gpgv";
1433          return;
1434       }
1435    }
1436
1437    if (Complete == true)
1438    {
1439       string FinalFile = _config->FindDir("Dir::State::lists");
1440       FinalFile += URItoFileName(RealURI);
1441       if (SigFile == DestFile)
1442          SigFile = FinalFile;
1443       Rename(DestFile,FinalFile);
1444       chmod(FinalFile.c_str(),0644);
1445       DestFile = FinalFile;
1446    }
1447 }
1448                                                                         /*}}}*/
1449 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1450 {
1451    // We have just finished downloading a Release file (it is not
1452    // verified yet)
1453
1454    string FileName = LookupTag(Message,"Filename");
1455    if (FileName.empty() == true)
1456    {
1457       Status = StatError;
1458       ErrorText = "Method gave a blank filename";
1459       return;
1460    }
1461
1462    if (FileName != DestFile)
1463    {
1464       Local = true;
1465       Desc.URI = "copy:" + FileName;
1466       QueueURI(Desc);
1467       return;
1468    }
1469
1470    // make sure to verify against the right file on I-M-S hit
1471    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1472    if(IMSHit)
1473    {
1474       string FinalFile = _config->FindDir("Dir::State::lists");
1475       FinalFile += URItoFileName(RealURI);
1476       if (SigFile == DestFile)
1477       {
1478          SigFile = FinalFile;
1479          // constructor of pkgAcqMetaClearSig moved it out of the way,
1480          // now move it back in on IMS hit for the 'old' file
1481          string const OldClearSig = DestFile + ".reverify";
1482          if (RealFileExists(OldClearSig) == true)
1483             Rename(OldClearSig, FinalFile);
1484       }
1485       DestFile = FinalFile;
1486    }
1487    Complete = true;
1488 }
1489                                                                         /*}}}*/
1490 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1491 {
1492    // At this point, the gpgv method has succeeded, so there is a
1493    // valid signature from a key in the trusted keyring.  We
1494    // perform additional verification of its contents, and use them
1495    // to verify the indexes we are about to download
1496
1497    if (!MetaIndexParser->Load(DestFile))
1498    {
1499       Status = StatAuthError;
1500       ErrorText = MetaIndexParser->ErrorText;
1501       return;
1502    }
1503
1504    if (!VerifyVendor(Message))
1505    {
1506       return;
1507    }
1508
1509    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1510       std::cerr << "Signature verification succeeded: "
1511                 << DestFile << std::endl;
1512
1513    // Download further indexes with verification
1514    QueueIndexes(true);
1515
1516    // is it a clearsigned MetaIndex file?
1517    if (DestFile == SigFile)
1518       return;
1519
1520    // Done, move signature file into position
1521    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1522       URItoFileName(RealURI) + ".gpg";
1523    Rename(SigFile,VerifiedSigFile);
1524    chmod(VerifiedSigFile.c_str(),0644);
1525 }
1526                                                                         /*}}}*/
1527 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1528 {
1529 #if 0
1530    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1531     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1532    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1533    {
1534       Status = StatError;
1535       ErrorText = MetaIndexParser->ErrorText;
1536       return;
1537    }
1538 #endif
1539    bool transInRelease = false;
1540    {
1541       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1542       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1543          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1544          if (k->find("Translation-") != std::string::npos)
1545          {
1546             transInRelease = true;
1547             break;
1548          }
1549    }
1550
1551    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1552         Target != IndexTargets->end();
1553         ++Target)
1554    {
1555       HashString ExpectedIndexHash;
1556       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1557       bool compressedAvailable = false;
1558       if (Record == NULL)
1559       {
1560          if ((*Target)->IsOptional() == true)
1561          {
1562             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1563             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1564                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1565                {
1566                   compressedAvailable = true;
1567                   break;
1568                }
1569          }
1570          else if (verify == true)
1571          {
1572             Status = StatAuthError;
1573             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1574             return;
1575          }
1576       }
1577       else
1578       {
1579          ExpectedIndexHash = Record->Hash;
1580          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1581          {
1582             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1583             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1584             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1585          }
1586          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1587          {
1588             Status = StatAuthError;
1589             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1590             return;
1591          }
1592       }
1593
1594       if ((*Target)->IsOptional() == true)
1595       {
1596          if ((*Target)->IsSubIndex() == true)
1597             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1598                                 (*Target)->ShortDesc, ExpectedIndexHash);
1599          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1600          {
1601             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1602                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1603                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1604                                    (*Target)->ShortDesc, ExpectedIndexHash);
1605             else
1606                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1607          }
1608          continue;
1609       }
1610
1611       /* Queue Packages file (either diff or full packages files, depending
1612          on the users option) - we also check if the PDiff Index file is listed
1613          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1614          instead, but passing the required info to it is to much hassle */
1615       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1616           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1617          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1618                              (*Target)->ShortDesc, ExpectedIndexHash);
1619       else
1620          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1621    }
1622 }
1623                                                                         /*}}}*/
1624 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1625 {
1626    string::size_type pos;
1627
1628    // check for missing sigs (that where not fatal because otherwise we had
1629    // bombed earlier)
1630    string missingkeys;
1631    string msg = _("There is no public key available for the "
1632                   "following key IDs:\n");
1633    pos = Message.find("NO_PUBKEY ");
1634    if (pos != std::string::npos)
1635    {
1636       string::size_type start = pos+strlen("NO_PUBKEY ");
1637       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1638       missingkeys += (Fingerprint);
1639    }
1640    if(!missingkeys.empty())
1641       _error->Warning("%s", (msg + missingkeys).c_str());
1642
1643    string Transformed = MetaIndexParser->GetExpectedDist();
1644
1645    if (Transformed == "../project/experimental")
1646    {
1647       Transformed = "experimental";
1648    }
1649
1650    pos = Transformed.rfind('/');
1651    if (pos != string::npos)
1652    {
1653       Transformed = Transformed.substr(0, pos);
1654    }
1655
1656    if (Transformed == ".")
1657    {
1658       Transformed = "";
1659    }
1660
1661    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1662        MetaIndexParser->GetValidUntil() > 0) {
1663       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1664       if (invalid_since > 0)
1665          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1666          // the time since then the file is invalid - formated in the same way as in
1667          // the download progress display (e.g. 7d 3h 42min 1s)
1668          return _error->Error(
1669             _("Release file for %s is expired (invalid since %s). "
1670               "Updates for this repository will not be applied."),
1671             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1672    }
1673
1674    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1675    {
1676       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1677       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1678       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1679    }
1680
1681    if (MetaIndexParser->CheckDist(Transformed) == false)
1682    {
1683       // This might become fatal one day
1684 //       Status = StatAuthError;
1685 //       ErrorText = "Conflicting distribution; expected "
1686 //          + MetaIndexParser->GetExpectedDist() + " but got "
1687 //          + MetaIndexParser->GetDist();
1688 //       return false;
1689       if (!Transformed.empty())
1690       {
1691          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1692                          Desc.Description.c_str(),
1693                          Transformed.c_str(),
1694                          MetaIndexParser->GetDist().c_str());
1695       }
1696    }
1697
1698    return true;
1699 }
1700                                                                         /*}}}*/
1701 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1702 // ---------------------------------------------------------------------
1703 /* */
1704 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1705 {
1706    if (AuthPass == true)
1707    {
1708       // gpgv method failed, if we have a good signature
1709       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1710       if (DestFile != SigFile)
1711          LastGoodSigFile.append(".gpg");
1712       LastGoodSigFile.append(".reverify");
1713
1714       if(FileExists(LastGoodSigFile))
1715       {
1716          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1717          if (DestFile != SigFile)
1718             VerifiedSigFile.append(".gpg");
1719          Rename(LastGoodSigFile, VerifiedSigFile);
1720          Status = StatTransientNetworkError;
1721          _error->Warning(_("An error occurred during the signature "
1722                            "verification. The repository is not updated "
1723                            "and the previous index files will be used. "
1724                            "GPG error: %s: %s\n"),
1725                          Desc.Description.c_str(),
1726                          LookupTag(Message,"Message").c_str());
1727          RunScripts("APT::Update::Auth-Failure");
1728          return;
1729       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1730          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1731          _error->Error(_("GPG error: %s: %s"),
1732                          Desc.Description.c_str(),
1733                          LookupTag(Message,"Message").c_str());
1734          return;
1735       } else {
1736          _error->Warning(_("GPG error: %s: %s"),
1737                          Desc.Description.c_str(),
1738                          LookupTag(Message,"Message").c_str());
1739       }
1740       // gpgv method failed
1741       ReportMirrorFailure("GPGFailure");
1742    }
1743
1744    /* Always move the meta index, even if gpgv failed. This ensures
1745     * that PackageFile objects are correctly filled in */
1746    if (FileExists(DestFile)) {
1747       string FinalFile = _config->FindDir("Dir::State::lists");
1748       FinalFile += URItoFileName(RealURI);
1749       /* InRelease files become Release files, otherwise
1750        * they would be considered as trusted later on */
1751       if (SigFile == DestFile) {
1752          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1753                                        "Release");
1754          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1755                                        "Release");
1756          SigFile = FinalFile;
1757       }
1758       Rename(DestFile,FinalFile);
1759       chmod(FinalFile.c_str(),0644);
1760
1761       DestFile = FinalFile;
1762    }
1763
1764    // No Release file was present, or verification failed, so fall
1765    // back to queueing Packages files without verification
1766    QueueIndexes(false);
1767 }
1768                                                                         /*}}}*/
1769 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1770                 string const &URI, string const &URIDesc, string const &ShortDesc,
1771                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1772                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1773                 const vector<struct IndexTarget*>* IndexTargets,
1774                 indexRecords* MetaIndexParser) :
1775         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1776         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1777         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1778 {
1779    SigFile = DestFile;
1780
1781    // keep the old InRelease around in case of transistent network errors
1782    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1783    if (RealFileExists(Final) == true)
1784    {
1785       string const LastGoodSig = DestFile + ".reverify";
1786       Rename(Final,LastGoodSig);
1787    }
1788 }
1789                                                                         /*}}}*/
1790 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1791 {
1792    // if the file was never queued undo file-changes done in the constructor
1793    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1794    {
1795       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1796       string const LastGoodSig = DestFile + ".reverify";
1797       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1798          Rename(LastGoodSig, Final);
1799    }
1800 }
1801                                                                         /*}}}*/
1802 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1803 // ---------------------------------------------------------------------
1804 // FIXME: this can go away once the InRelease file is used widely
1805 string pkgAcqMetaClearSig::Custom600Headers()
1806 {
1807    string Final = _config->FindDir("Dir::State::lists");
1808    Final += URItoFileName(RealURI);
1809
1810    struct stat Buf;
1811    if (stat(Final.c_str(),&Buf) != 0)
1812    {
1813       Final = DestFile + ".reverify";
1814       if (stat(Final.c_str(),&Buf) != 0)
1815          return "\nIndex-File: true\nFail-Ignore: true\n";
1816    }
1817
1818    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1819 }
1820                                                                         /*}}}*/
1821 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1822 {
1823    if (AuthPass == false)
1824    {
1825       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1826       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1827       string FinalFile = _config->FindDir("Dir::State::lists");
1828       FinalFile.append(URItoFileName(RealURI));
1829       if (FileExists(FinalFile))
1830          unlink(FinalFile.c_str());
1831
1832       new pkgAcqMetaSig(Owner,
1833                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1834                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1835                         IndexTargets, MetaIndexParser);
1836       if (Cnf->LocalOnly == true ||
1837           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1838          Dequeue();
1839    }
1840    else
1841       pkgAcqMetaIndex::Failed(Message, Cnf);
1842 }
1843                                                                         /*}}}*/
1844 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1845 // ---------------------------------------------------------------------
1846 /* This just sets up the initial fetch environment and queues the first
1847    possibilitiy */
1848 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1849                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1850                              string &StoreFilename) :
1851                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1852                StoreFilename(StoreFilename), Vf(Version.FileList()),
1853                Trusted(false)
1854 {
1855    Retries = _config->FindI("Acquire::Retries",0);
1856
1857    if (Version.Arch() == 0)
1858    {
1859       _error->Error(_("I wasn't able to locate a file for the %s package. "
1860                       "This might mean you need to manually fix this package. "
1861                       "(due to missing arch)"),
1862                     Version.ParentPkg().FullName().c_str());
1863       return;
1864    }
1865
1866    /* We need to find a filename to determine the extension. We make the
1867       assumption here that all the available sources for this version share
1868       the same extension.. */
1869    // Skip not source sources, they do not have file fields.
1870    for (; Vf.end() == false; ++Vf)
1871    {
1872       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1873          continue;
1874       break;
1875    }
1876
1877    // Does not really matter here.. we are going to fail out below
1878    if (Vf.end() != true)
1879    {
1880       // If this fails to get a file name we will bomb out below.
1881       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1882       if (_error->PendingError() == true)
1883          return;
1884
1885       // Generate the final file name as: package_version_arch.foo
1886       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1887                       QuoteString(Version.VerStr(),"_:") + '_' +
1888                       QuoteString(Version.Arch(),"_:.") +
1889                       "." + flExtension(Parse.FileName());
1890    }
1891
1892    // check if we have one trusted source for the package. if so, switch
1893    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1894    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1895    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1896    bool seenUntrusted = false;
1897    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1898    {
1899       pkgIndexFile *Index;
1900       if (Sources->FindIndex(i.File(),Index) == false)
1901          continue;
1902
1903       if (debugAuth == true)
1904          std::cerr << "Checking index: " << Index->Describe()
1905                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1906
1907       if (Index->IsTrusted() == true)
1908       {
1909          Trusted = true;
1910          if (allowUnauth == false)
1911             break;
1912       }
1913       else
1914          seenUntrusted = true;
1915    }
1916
1917    // "allow-unauthenticated" restores apts old fetching behaviour
1918    // that means that e.g. unauthenticated file:// uris are higher
1919    // priority than authenticated http:// uris
1920    if (allowUnauth == true && seenUntrusted == true)
1921       Trusted = false;
1922
1923    // Select a source
1924    if (QueueNext() == false && _error->PendingError() == false)
1925       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1926                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1927 }
1928                                                                         /*}}}*/
1929 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1930 // ---------------------------------------------------------------------
1931 /* This queues the next available file version for download. It checks if
1932    the archive is already available in the cache and stashs the MD5 for
1933    checking later. */
1934 bool pkgAcqArchive::QueueNext()
1935 {
1936    string const ForceHash = _config->Find("Acquire::ForceHash");
1937    for (; Vf.end() == false; ++Vf)
1938    {
1939       // Ignore not source sources
1940       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1941          continue;
1942
1943       // Try to cross match against the source list
1944       pkgIndexFile *Index;
1945       if (Sources->FindIndex(Vf.File(),Index) == false)
1946             continue;
1947
1948       // only try to get a trusted package from another source if that source
1949       // is also trusted
1950       if(Trusted && !Index->IsTrusted())
1951          continue;
1952
1953       // Grab the text package record
1954       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1955       if (_error->PendingError() == true)
1956          return false;
1957
1958       string PkgFile = Parse.FileName();
1959       if (ForceHash.empty() == false)
1960       {
1961          if(stringcasecmp(ForceHash, "sha512") == 0)
1962             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1963          else if(stringcasecmp(ForceHash, "sha256") == 0)
1964             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1965          else if (stringcasecmp(ForceHash, "sha1") == 0)
1966             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1967          else
1968             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1969       }
1970       else
1971       {
1972          string Hash;
1973          if ((Hash = Parse.SHA512Hash()).empty() == false)
1974             ExpectedHash = HashString("SHA512", Hash);
1975          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1976             ExpectedHash = HashString("SHA256", Hash);
1977          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1978             ExpectedHash = HashString("SHA1", Hash);
1979          else
1980             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1981       }
1982       if (PkgFile.empty() == true)
1983          return _error->Error(_("The package index files are corrupted. No Filename: "
1984                               "field for package %s."),
1985                               Version.ParentPkg().Name());
1986
1987       Desc.URI = Index->ArchiveURI(PkgFile);
1988       Desc.Description = Index->ArchiveInfo(Version);
1989       Desc.Owner = this;
1990       Desc.ShortDesc = Version.ParentPkg().FullName(true);
1991
1992       // See if we already have the file. (Legacy filenames)
1993       FileSize = Version->Size;
1994       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1995       struct stat Buf;
1996       if (stat(FinalFile.c_str(),&Buf) == 0)
1997       {
1998          // Make sure the size matches
1999          if ((unsigned long long)Buf.st_size == Version->Size)
2000          {
2001             Complete = true;
2002             Local = true;
2003             Status = StatDone;
2004             StoreFilename = DestFile = FinalFile;
2005             return true;
2006          }
2007
2008          /* Hmm, we have a file and its size does not match, this means it is
2009             an old style mismatched arch */
2010          unlink(FinalFile.c_str());
2011       }
2012
2013       // Check it again using the new style output filenames
2014       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2015       if (stat(FinalFile.c_str(),&Buf) == 0)
2016       {
2017          // Make sure the size matches
2018          if ((unsigned long long)Buf.st_size == Version->Size)
2019          {
2020             Complete = true;
2021             Local = true;
2022             Status = StatDone;
2023             StoreFilename = DestFile = FinalFile;
2024             return true;
2025          }
2026
2027          /* Hmm, we have a file and its size does not match, this shouldn't
2028             happen.. */
2029          unlink(FinalFile.c_str());
2030       }
2031
2032       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2033
2034       // Check the destination file
2035       if (stat(DestFile.c_str(),&Buf) == 0)
2036       {
2037          // Hmm, the partial file is too big, erase it
2038          if ((unsigned long long)Buf.st_size > Version->Size)
2039             unlink(DestFile.c_str());
2040          else
2041             PartialSize = Buf.st_size;
2042       }
2043
2044       // Disables download of archives - useful if no real installation follows,
2045       // e.g. if we are just interested in proposed installation order
2046       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2047       {
2048          Complete = true;
2049          Local = true;
2050          Status = StatDone;
2051          StoreFilename = DestFile = FinalFile;
2052          return true;
2053       }
2054
2055       // Create the item
2056       Local = false;
2057       QueueURI(Desc);
2058
2059       ++Vf;
2060       return true;
2061    }
2062    return false;
2063 }
2064                                                                         /*}}}*/
2065 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2066 // ---------------------------------------------------------------------
2067 /* */
2068 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2069                          pkgAcquire::MethodConfig *Cfg)
2070 {
2071    Item::Done(Message,Size,CalcHash,Cfg);
2072
2073    // Check the size
2074    if (Size != Version->Size)
2075    {
2076       RenameOnError(SizeMismatch);
2077       return;
2078    }
2079
2080    // Check the hash
2081    if(ExpectedHash.toStr() != CalcHash)
2082    {
2083       RenameOnError(HashSumMismatch);
2084       return;
2085    }
2086
2087    // Grab the output filename
2088    string FileName = LookupTag(Message,"Filename");
2089    if (FileName.empty() == true)
2090    {
2091       Status = StatError;
2092       ErrorText = "Method gave a blank filename";
2093       return;
2094    }
2095
2096    Complete = true;
2097
2098    // Reference filename
2099    if (FileName != DestFile)
2100    {
2101       StoreFilename = DestFile = FileName;
2102       Local = true;
2103       return;
2104    }
2105
2106    // Done, move it into position
2107    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2108    FinalFile += flNotDir(StoreFilename);
2109    Rename(DestFile,FinalFile);
2110
2111    StoreFilename = DestFile = FinalFile;
2112    Complete = true;
2113 }
2114                                                                         /*}}}*/
2115 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2116 // ---------------------------------------------------------------------
2117 /* Here we try other sources */
2118 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2119 {
2120    ErrorText = LookupTag(Message,"Message");
2121
2122    /* We don't really want to retry on failed media swaps, this prevents
2123       that. An interesting observation is that permanent failures are not
2124       recorded. */
2125    if (Cnf->Removable == true &&
2126        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2127    {
2128       // Vf = Version.FileList();
2129       while (Vf.end() == false) ++Vf;
2130       StoreFilename = string();
2131       Item::Failed(Message,Cnf);
2132       return;
2133    }
2134
2135    if (QueueNext() == false)
2136    {
2137       // This is the retry counter
2138       if (Retries != 0 &&
2139           Cnf->LocalOnly == false &&
2140           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2141       {
2142          Retries--;
2143          Vf = Version.FileList();
2144          if (QueueNext() == true)
2145             return;
2146       }
2147
2148       StoreFilename = string();
2149       Item::Failed(Message,Cnf);
2150    }
2151 }
2152                                                                         /*}}}*/
2153 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2154 // ---------------------------------------------------------------------
2155 APT_PURE bool pkgAcqArchive::IsTrusted()
2156 {
2157    return Trusted;
2158 }
2159                                                                         /*}}}*/
2160 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2161 // ---------------------------------------------------------------------
2162 /* */
2163 void pkgAcqArchive::Finished()
2164 {
2165    if (Status == pkgAcquire::Item::StatDone &&
2166        Complete == true)
2167       return;
2168    StoreFilename = string();
2169 }
2170                                                                         /*}}}*/
2171 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2172 // ---------------------------------------------------------------------
2173 /* The file is added to the queue */
2174 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2175                        unsigned long long Size,string Dsc,string ShortDesc,
2176                        const string &DestDir, const string &DestFilename,
2177                        bool IsIndexFile) :
2178                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2179 {
2180    Retries = _config->FindI("Acquire::Retries",0);
2181
2182    if(!DestFilename.empty())
2183       DestFile = DestFilename;
2184    else if(!DestDir.empty())
2185       DestFile = DestDir + "/" + flNotDir(URI);
2186    else
2187       DestFile = flNotDir(URI);
2188
2189    // Create the item
2190    Desc.URI = URI;
2191    Desc.Description = Dsc;
2192    Desc.Owner = this;
2193
2194    // Set the short description to the archive component
2195    Desc.ShortDesc = ShortDesc;
2196
2197    // Get the transfer sizes
2198    FileSize = Size;
2199    struct stat Buf;
2200    if (stat(DestFile.c_str(),&Buf) == 0)
2201    {
2202       // Hmm, the partial file is too big, erase it
2203       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2204          unlink(DestFile.c_str());
2205       else
2206          PartialSize = Buf.st_size;
2207    }
2208
2209    QueueURI(Desc);
2210 }
2211                                                                         /*}}}*/
2212 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2213 // ---------------------------------------------------------------------
2214 /* */
2215 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2216                       pkgAcquire::MethodConfig *Cnf)
2217 {
2218    Item::Done(Message,Size,CalcHash,Cnf);
2219
2220    // Check the hash
2221    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2222    {
2223       RenameOnError(HashSumMismatch);
2224       return;
2225    }
2226
2227    string FileName = LookupTag(Message,"Filename");
2228    if (FileName.empty() == true)
2229    {
2230       Status = StatError;
2231       ErrorText = "Method gave a blank filename";
2232       return;
2233    }
2234
2235    Complete = true;
2236
2237    // The files timestamp matches
2238    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2239       return;
2240
2241    // We have to copy it into place
2242    if (FileName != DestFile)
2243    {
2244       Local = true;
2245       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2246           Cnf->Removable == true)
2247       {
2248          Desc.URI = "copy:" + FileName;
2249          QueueURI(Desc);
2250          return;
2251       }
2252
2253       // Erase the file if it is a symlink so we can overwrite it
2254       struct stat St;
2255       if (lstat(DestFile.c_str(),&St) == 0)
2256       {
2257          if (S_ISLNK(St.st_mode) != 0)
2258             unlink(DestFile.c_str());
2259       }
2260
2261       // Symlink the file
2262       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2263       {
2264          ErrorText = "Link to " + DestFile + " failure ";
2265          Status = StatError;
2266          Complete = false;
2267       }
2268    }
2269 }
2270                                                                         /*}}}*/
2271 // AcqFile::Failed - Failure handler                                    /*{{{*/
2272 // ---------------------------------------------------------------------
2273 /* Here we try other sources */
2274 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2275 {
2276    ErrorText = LookupTag(Message,"Message");
2277
2278    // This is the retry counter
2279    if (Retries != 0 &&
2280        Cnf->LocalOnly == false &&
2281        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2282    {
2283       Retries--;
2284       QueueURI(Desc);
2285       return;
2286    }
2287
2288    Item::Failed(Message,Cnf);
2289 }
2290                                                                         /*}}}*/
2291 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2292 // ---------------------------------------------------------------------
2293 /* The only header we use is the last-modified header. */
2294 string pkgAcqFile::Custom600Headers()
2295 {
2296    if (IsIndexFile)
2297       return "\nIndex-File: true";
2298    return "";
2299 }
2300                                                                         /*}}}*/