apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::Item::Item - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  57                        PartialSize(0), Mode(0), ID(0), Complete(false),
  58                        Local(false), QueueCounter(0)
  59 {
  60    Owner->Add(this);
  61    Status = StatIdle;
  62 }
  63                                                                         /*}}}*/
  64 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 pkgAcquire::Item::~Item()
  68 {
  69    Owner->Remove(this);
  70 }
  71                                                                         /*}}}*/
  72 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  73 // ---------------------------------------------------------------------
  74 /* We return to an idle state if there are still other queues that could
  75    fetch this object */
  76 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  77 {
  78    Status = StatIdle;
  79    ErrorText = LookupTag(Message,"Message");
  80    UsedMirror =  LookupTag(Message,"UsedMirror");
  81    if (QueueCounter <= 1)
  82    {
  83       /* This indicates that the file is not available right now but might
  84          be sometime later. If we do a retry cycle then this should be
  85          retried [CDROMs] */
  86       if (Cnf->LocalOnly == true &&
  87           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  88       {
  89          Status = StatIdle;
  90          Dequeue();
  91          return;
  92       }
  93
  94       Status = StatError;
  95       Dequeue();
  96    }
  97
  98    // report mirror failure back to LP if we actually use a mirror
  99    string FailReason = LookupTag(Message, "FailReason");
 100    if(FailReason.size() != 0)
 101       ReportMirrorFailure(FailReason);
 102    else
 103       ReportMirrorFailure(ErrorText);
 104 }
 105                                                                         /*}}}*/
 106 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* Stash status and the file size. Note that setting Complete means
 109    sub-phases of the acquire process such as decompresion are operating */
 110 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 111 {
 112    Status = StatFetching;
 113    if (FileSize == 0 && Complete == false)
 114       FileSize = Size;
 115 }
 116                                                                         /*}}}*/
 117 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/,
 121                             pkgAcquire::MethodConfig * /*Cnf*/)
 122 {
 123    // We just downloaded something..
 124    string FileName = LookupTag(Message,"Filename");
 125    UsedMirror =  LookupTag(Message,"UsedMirror");
 126    if (Complete == false && !Local && FileName == DestFile)
 127    {
 128       if (Owner->Log != 0)
 129          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 130    }
 131
 132    if (FileSize == 0)
 133       FileSize= Size;
 134    Status = StatDone;
 135    ErrorText = string();
 136    Owner->Dequeue(this);
 137 }
 138                                                                         /*}}}*/
 139 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This helper function is used by a lot of item methods as their final
 142    step */
 143 void pkgAcquire::Item::Rename(string From,string To)
 144 {
 145    if (rename(From.c_str(),To.c_str()) != 0)
 146    {
 147       char S[300];
 148       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 149               From.c_str(),To.c_str());
 150       Status = StatError;
 151       ErrorText = S;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 156 {
 157    if(FileExists(DestFile))
 158       Rename(DestFile, DestFile + ".FAILED");
 159
 160    switch (error)
 161    {
 162       case HashSumMismatch:
 163          ErrorText = _("Hash Sum mismatch");
 164          Status = StatAuthError;
 165          ReportMirrorFailure("HashChecksumFailure");
 166          break;
 167       case SizeMismatch:
 168          ErrorText = _("Size mismatch");
 169          Status = StatAuthError;
 170          ReportMirrorFailure("SizeFailure");
 171          break;
 172       case InvalidFormat:
 173          ErrorText = _("Invalid file format");
 174          Status = StatError;
 175          // do not report as usually its not the mirrors fault, but Portal/Proxy
 176          break;
 177    }
 178    return false;
 179 }
 180                                                                         /*}}}*/
 181 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 182 // ---------------------------------------------------------------------
 183 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 184 {
 185    // we only act if a mirror was used at all
 186    if(UsedMirror.empty())
 187       return;
 188 #if 0
 189    std::cerr << "\nReportMirrorFailure: "
 190              << UsedMirror
 191              << " Uri: " << DescURI()
 192              << " FailCode: "
 193              << FailCode << std::endl;
 194 #endif
 195    const char *Args[40];
 196    unsigned int i = 0;
 197    string report = _config->Find("Methods::Mirror::ProblemReporting",
 198                                  "/usr/lib/apt/apt-report-mirror-failure");
 199    if(!FileExists(report))
 200       return;
 201    Args[i++] = report.c_str();
 202    Args[i++] = UsedMirror.c_str();
 203    Args[i++] = DescURI().c_str();
 204    Args[i++] = FailCode.c_str();
 205    Args[i++] = NULL;
 206    pid_t pid = ExecFork();
 207    if(pid < 0)
 208    {
 209       _error->Error("ReportMirrorFailure Fork failed");
 210       return;
 211    }
 212    else if(pid == 0)
 213    {
 214       execvp(Args[0], (char**)Args);
 215       std::cerr << "Could not exec " << Args[0] << std::endl;
 216       _exit(100);
 217    }
 218    if(!ExecWait(pid, "report-mirror-failure"))
 219    {
 220       _error->Warning("Couldn't report problem to '%s'",
 221                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 222    }
 223 }
 224                                                                         /*}}}*/
 225 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 226 // ---------------------------------------------------------------------
 227 /* Get a sub-index file based on checksums from a 'master' file and
 228    possibly query additional files */
 229 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 230                                  string const &URIDesc, string const &ShortDesc,
 231                                  HashString const &ExpectedHash)
 232    : Item(Owner), ExpectedHash(ExpectedHash)
 233 {
 234    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 235    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 236
 237    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 238    DestFile += URItoFileName(URI);
 239
 240    Desc.URI = URI;
 241    Desc.Description = URIDesc;
 242    Desc.Owner = this;
 243    Desc.ShortDesc = ShortDesc;
 244
 245    QueueURI(Desc);
 246
 247    if(Debug)
 248       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 249 }
 250                                                                         /*}}}*/
 251 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 252 // ---------------------------------------------------------------------
 253 /* The only header we use is the last-modified header. */
 254 string pkgAcqSubIndex::Custom600Headers()
 255 {
 256    string Final = _config->FindDir("Dir::State::lists");
 257    Final += URItoFileName(Desc.URI);
 258
 259    struct stat Buf;
 260    if (stat(Final.c_str(),&Buf) != 0)
 261       return "\nIndex-File: true\nFail-Ignore: true\n";
 262    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 263 }
 264                                                                         /*}}}*/
 265 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 266 {
 267    if(Debug)
 268       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 269
 270    Complete = false;
 271    Status = StatDone;
 272    Dequeue();
 273
 274    // No good Index is provided
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 278                            pkgAcquire::MethodConfig *Cnf)
 279 {
 280    if(Debug)
 281       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 282
 283    string FileName = LookupTag(Message,"Filename");
 284    if (FileName.empty() == true)
 285    {
 286       Status = StatError;
 287       ErrorText = "Method gave a blank filename";
 288       return;
 289    }
 290
 291    if (FileName != DestFile)
 292    {
 293       Local = true;
 294       Desc.URI = "copy:" + FileName;
 295       QueueURI(Desc);
 296       return;
 297    }
 298
 299    Item::Done(Message,Size,Md5Hash,Cnf);
 300
 301    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 302
 303    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 304    indexRecords SubIndexParser;
 305    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 306       Status = StatError;
 307       ErrorText = SubIndexParser.ErrorText;
 308       return;
 309    }
 310
 311    // success in downloading the index
 312    // rename the index
 313    if(Debug)
 314       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 315    Rename(DestFile,FinalFile);
 316    chmod(FinalFile.c_str(),0644);
 317    DestFile = FinalFile;
 318
 319    if(ParseIndex(DestFile) == false)
 320       return Failed("", NULL);
 321
 322    Complete = true;
 323    Status = StatDone;
 324    Dequeue();
 325    return;
 326 }
 327                                                                         /*}}}*/
 328 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 329 {
 330    indexRecords SubIndexParser;
 331    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 332       return false;
 333    // so something with the downloaded index
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* Get the DiffIndex file first and see if there are patches available
 340  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 341  * patches. If anything goes wrong in that process, it will fall back to
 342  * the original packages file
 343  */
 344 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 345                                  string URI,string URIDesc,string ShortDesc,
 346                                  HashString ExpectedHash)
 347    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 348      Description(URIDesc)
 349 {
 350
 351    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 352
 353    Desc.Description = URIDesc + "/DiffIndex";
 354    Desc.Owner = this;
 355    Desc.ShortDesc = ShortDesc;
 356    Desc.URI = URI + ".diff/Index";
 357
 358    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 359    DestFile += URItoFileName(URI) + string(".DiffIndex");
 360
 361    if(Debug)
 362       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 363
 364    // look for the current package file
 365    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 366    CurrentPackagesFile += URItoFileName(RealURI);
 367
 368    // FIXME: this file:/ check is a hack to prevent fetching
 369    //        from local sources. this is really silly, and
 370    //        should be fixed cleanly as soon as possible
 371    if(!FileExists(CurrentPackagesFile) ||
 372       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 373    {
 374       // we don't have a pkg file or we don't want to queue
 375       if(Debug)
 376          std::clog << "No index file, local or canceld by user" << std::endl;
 377       Failed("", NULL);
 378       return;
 379    }
 380
 381    if(Debug)
 382       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 383          << CurrentPackagesFile << std::endl;
 384
 385    QueueURI(Desc);
 386
 387 }
 388                                                                         /*}}}*/
 389 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* The only header we use is the last-modified header. */
 392 string pkgAcqDiffIndex::Custom600Headers()
 393 {
 394    string Final = _config->FindDir("Dir::State::lists");
 395    Final += URItoFileName(RealURI) + string(".IndexDiff");
 396
 397    if(Debug)
 398       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 399
 400    struct stat Buf;
 401    if (stat(Final.c_str(),&Buf) != 0)
 402       return "\nIndex-File: true";
 403
 404    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 405 }
 406                                                                         /*}}}*/
 407 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 408 {
 409    if(Debug)
 410       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 411          << std::endl;
 412
 413    pkgTagSection Tags;
 414    string ServerSha1;
 415    vector<DiffInfo> available_patches;
 416
 417    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 418    pkgTagFile TF(&Fd);
 419    if (_error->PendingError() == true)
 420       return false;
 421
 422    if(TF.Step(Tags) == true)
 423    {
 424       bool found = false;
 425       DiffInfo d;
 426       string size;
 427
 428       string const tmp = Tags.FindS("SHA1-Current");
 429       std::stringstream ss(tmp);
 430       ss >> ServerSha1 >> size;
 431       unsigned long const ServerSize = atol(size.c_str());
 432
 433       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 434       SHA1Summation SHA1;
 435       SHA1.AddFD(fd);
 436       string const local_sha1 = SHA1.Result();
 437
 438       if(local_sha1 == ServerSha1)
 439       {
 440          // we have the same sha1 as the server so we are done here
 441          if(Debug)
 442             std::clog << "Package file is up-to-date" << std::endl;
 443          // list cleanup needs to know that this file as well as the already
 444          // present index is ours, so we create an empty diff to save it for us
 445          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 446                ExpectedHash, ServerSha1, available_patches);
 447          return true;
 448       }
 449       else
 450       {
 451          if(Debug)
 452             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 453
 454          // check the historie and see what patches we need
 455          string const history = Tags.FindS("SHA1-History");
 456          std::stringstream hist(history);
 457          while(hist >> d.sha1 >> size >> d.file)
 458          {
 459             // read until the first match is found
 460             // from that point on, we probably need all diffs
 461             if(d.sha1 == local_sha1)
 462                found=true;
 463             else if (found == false)
 464                continue;
 465
 466             if(Debug)
 467                std::clog << "Need to get diff: " << d.file << std::endl;
 468             available_patches.push_back(d);
 469          }
 470
 471          if (available_patches.empty() == false)
 472          {
 473             // patching with too many files is rather slow compared to a fast download
 474             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 475             if (fileLimit != 0 && fileLimit < available_patches.size())
 476             {
 477                if (Debug)
 478                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 479                         << ") so fallback to complete download" << std::endl;
 480                return false;
 481             }
 482
 483             // see if the patches are too big
 484             found = false; // it was true and it will be true again at the end
 485             d = *available_patches.begin();
 486             string const firstPatch = d.file;
 487             unsigned long patchesSize = 0;
 488             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 489             while(patches >> d.sha1 >> size >> d.file)
 490             {
 491                if (firstPatch == d.file)
 492                   found = true;
 493                else if (found == false)
 494                   continue;
 495
 496                patchesSize += atol(size.c_str());
 497             }
 498             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 499             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 500             {
 501                if (Debug)
 502                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 503                         << ") so fallback to complete download" << std::endl;
 504                return false;
 505             }
 506          }
 507       }
 508
 509       // we have something, queue the next diff
 510       if(found)
 511       {
 512          // queue the diffs
 513          string::size_type const last_space = Description.rfind(" ");
 514          if(last_space != string::npos)
 515             Description.erase(last_space, Description.size()-last_space);
 516
 517          /* decide if we should download patches one by one or in one go:
 518             The first is good if the server merges patches, but many don't so client
 519             based merging can be attempt in which case the second is better.
 520             "bad things" will happen if patches are merged on the server,
 521             but client side merging is attempt as well */
 522          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 523          if (pdiff_merge == true)
 524          {
 525             // reprepro adds this flag if it has merged patches on the server
 526             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 527             pdiff_merge = (precedence != "merged");
 528          }
 529
 530          if (pdiff_merge == false)
 531             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 532                   ExpectedHash, ServerSha1, available_patches);
 533          else
 534          {
 535             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 536             for(size_t i = 0; i < available_patches.size(); ++i)
 537                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 538                      available_patches[i], diffs);
 539          }
 540
 541          Complete = false;
 542          Status = StatDone;
 543          Dequeue();
 544          return true;
 545       }
 546    }
 547
 548    // Nothing found, report and return false
 549    // Failing here is ok, if we return false later, the full
 550    // IndexFile is queued
 551    if(Debug)
 552       std::clog << "Can't find a patch in the index file" << std::endl;
 553    return false;
 554 }
 555                                                                         /*}}}*/
 556 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 557 {
 558    if(Debug)
 559       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 560                 << "Falling back to normal index file acquire" << std::endl;
 561
 562    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 563                    ExpectedHash);
 564
 565    Complete = false;
 566    Status = StatDone;
 567    Dequeue();
 568 }
 569                                                                         /*}}}*/
 570 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 571                            pkgAcquire::MethodConfig *Cnf)
 572 {
 573    if(Debug)
 574       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 575
 576    Item::Done(Message,Size,Md5Hash,Cnf);
 577
 578    string FinalFile;
 579    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 580
 581    // success in downloading the index
 582    // rename the index
 583    FinalFile += string(".IndexDiff");
 584    if(Debug)
 585       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 586                 << std::endl;
 587    Rename(DestFile,FinalFile);
 588    chmod(FinalFile.c_str(),0644);
 589    DestFile = FinalFile;
 590
 591    if(!ParseDiffIndex(DestFile))
 592       return Failed("", NULL);
 593
 594    Complete = true;
 595    Status = StatDone;
 596    Dequeue();
 597    return;
 598 }
 599                                                                         /*}}}*/
 600 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* The package diff is added to the queue. one object is constructed
 603  * for each diff and the index
 604  */
 605 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 606                                    string URI,string URIDesc,string ShortDesc,
 607                                    HashString ExpectedHash,
 608                                    string ServerSha1,
 609                                    vector<DiffInfo> diffs)
 610    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 611      available_patches(diffs), ServerSha1(ServerSha1)
 612 {
 613
 614    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 615    DestFile += URItoFileName(URI);
 616
 617    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 618
 619    Description = URIDesc;
 620    Desc.Owner = this;
 621    Desc.ShortDesc = ShortDesc;
 622
 623    if(available_patches.empty() == true)
 624    {
 625       // we are done (yeah!)
 626       Finish(true);
 627    }
 628    else
 629    {
 630       // get the next diff
 631       State = StateFetchDiff;
 632       QueueNextDiff();
 633    }
 634 }
 635                                                                         /*}}}*/
 636 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 637 {
 638    if(Debug)
 639       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 640                 << "Falling back to normal index file acquire" << std::endl;
 641    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 642                    ExpectedHash);
 643    Finish();
 644 }
 645                                                                         /*}}}*/
 646 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 647 void pkgAcqIndexDiffs::Finish(bool allDone)
 648 {
 649    // we restore the original name, this is required, otherwise
 650    // the file will be cleaned
 651    if(allDone)
 652    {
 653       DestFile = _config->FindDir("Dir::State::lists");
 654       DestFile += URItoFileName(RealURI);
 655
 656       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 657       {
 658          RenameOnError(HashSumMismatch);
 659          Dequeue();
 660          return;
 661       }
 662
 663       // this is for the "real" finish
 664       Complete = true;
 665       Status = StatDone;
 666       Dequeue();
 667       if(Debug)
 668          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 669       return;
 670    }
 671
 672    if(Debug)
 673       std::clog << "Finishing: " << Desc.URI << std::endl;
 674    Complete = false;
 675    Status = StatDone;
 676    Dequeue();
 677    return;
 678 }
 679                                                                         /*}}}*/
 680 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 681 {
 682
 683    // calc sha1 of the just patched file
 684    string FinalFile = _config->FindDir("Dir::State::lists");
 685    FinalFile += URItoFileName(RealURI);
 686
 687    FileFd fd(FinalFile, FileFd::ReadOnly);
 688    SHA1Summation SHA1;
 689    SHA1.AddFD(fd);
 690    string local_sha1 = string(SHA1.Result());
 691    if(Debug)
 692       std::clog << "QueueNextDiff: "
 693                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 694
 695    // final file reached before all patches are applied
 696    if(local_sha1 == ServerSha1)
 697    {
 698       Finish(true);
 699       return true;
 700    }
 701
 702    // remove all patches until the next matching patch is found
 703    // this requires the Index file to be ordered
 704    for(vector<DiffInfo>::iterator I=available_patches.begin();
 705        available_patches.empty() == false &&
 706           I != available_patches.end() &&
 707           I->sha1 != local_sha1;
 708        ++I)
 709    {
 710       available_patches.erase(I);
 711    }
 712
 713    // error checking and falling back if no patch was found
 714    if(available_patches.empty() == true)
 715    {
 716       Failed("", NULL);
 717       return false;
 718    }
 719
 720    // queue the right diff
 721    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 722    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 723    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 724    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 725
 726    if(Debug)
 727       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 728
 729    QueueURI(Desc);
 730
 731    return true;
 732 }
 733                                                                         /*}}}*/
 734 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 735                             pkgAcquire::MethodConfig *Cnf)
 736 {
 737    if(Debug)
 738       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 739
 740    Item::Done(Message,Size,Md5Hash,Cnf);
 741
 742    string FinalFile;
 743    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 744
 745    // success in downloading a diff, enter ApplyDiff state
 746    if(State == StateFetchDiff)
 747    {
 748
 749       // rred excepts the patch as $FinalFile.ed
 750       Rename(DestFile,FinalFile+".ed");
 751
 752       if(Debug)
 753          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 754
 755       State = StateApplyDiff;
 756       Local = true;
 757       Desc.URI = "rred:" + FinalFile;
 758       QueueURI(Desc);
 759       Mode = "rred";
 760       return;
 761    }
 762
 763
 764    // success in download/apply a diff, queue next (if needed)
 765    if(State == StateApplyDiff)
 766    {
 767       // remove the just applied patch
 768       available_patches.erase(available_patches.begin());
 769       unlink((FinalFile + ".ed").c_str());
 770
 771       // move into place
 772       if(Debug)
 773       {
 774          std::clog << "Moving patched file in place: " << std::endl
 775                    << DestFile << " -> " << FinalFile << std::endl;
 776       }
 777       Rename(DestFile,FinalFile);
 778       chmod(FinalFile.c_str(),0644);
 779
 780       // see if there is more to download
 781       if(available_patches.empty() == false) {
 782          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 783                               ExpectedHash, ServerSha1, available_patches);
 784          return Finish();
 785       } else
 786          return Finish(true);
 787    }
 788 }
 789                                                                         /*}}}*/
 790 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 791 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 792                                    string const &URI, string const &URIDesc,
 793                                    string const &ShortDesc, HashString const &ExpectedHash,
 794                                    DiffInfo const &patch,
 795                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 796    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 797      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 798 {
 799
 800    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 801    DestFile += URItoFileName(URI);
 802
 803    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 804
 805    Description = URIDesc;
 806    Desc.Owner = this;
 807    Desc.ShortDesc = ShortDesc;
 808
 809    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 810    Desc.Description = Description + " " + patch.file + string(".pdiff");
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 813
 814    if(Debug)
 815       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 816
 817    QueueURI(Desc);
 818 }
 819                                                                         /*}}}*/
 820 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 821 {
 822    if(Debug)
 823       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 824    Complete = false;
 825    Status = StatDone;
 826    Dequeue();
 827
 828    // check if we are the first to fail, otherwise we are done here
 829    State = StateDoneDiff;
 830    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 831          I != allPatches->end(); ++I)
 832       if ((*I)->State == StateErrorDiff)
 833          return;
 834
 835    // first failure means we should fallback
 836    State = StateErrorDiff;
 837    std::clog << "Falling back to normal index file acquire" << std::endl;
 838    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 839                    ExpectedHash);
 840 }
 841                                                                         /*}}}*/
 842 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 843                             pkgAcquire::MethodConfig *Cnf)
 844 {
 845    if(Debug)
 846       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 847
 848    Item::Done(Message,Size,Md5Hash,Cnf);
 849
 850    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 851
 852    if (State == StateFetchDiff)
 853    {
 854       // rred expects the patch as $FinalFile.ed.$patchname.gz
 855       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 856
 857       // check if this is the last completed diff
 858       State = StateDoneDiff;
 859       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 860             I != allPatches->end(); ++I)
 861          if ((*I)->State != StateDoneDiff)
 862          {
 863             if(Debug)
 864                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 865             return;
 866          }
 867
 868       // this is the last completed diff, so we are ready to apply now
 869       State = StateApplyDiff;
 870
 871       if(Debug)
 872          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 873
 874       Local = true;
 875       Desc.URI = "rred:" + FinalFile;
 876       QueueURI(Desc);
 877       Mode = "rred";
 878       return;
 879    }
 880    // success in download/apply all diffs, clean up
 881    else if (State == StateApplyDiff)
 882    {
 883       // see if we really got the expected file
 884       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 885       {
 886          RenameOnError(HashSumMismatch);
 887          return;
 888       }
 889
 890       // move the result into place
 891       if(Debug)
 892          std::clog << "Moving patched file in place: " << std::endl
 893                    << DestFile << " -> " << FinalFile << std::endl;
 894       Rename(DestFile, FinalFile);
 895       chmod(FinalFile.c_str(), 0644);
 896
 897       // otherwise lists cleanup will eat the file
 898       DestFile = FinalFile;
 899
 900       // ensure the ed's are gone regardless of list-cleanup
 901       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 902             I != allPatches->end(); ++I)
 903       {
 904             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 905             unlink(patch.c_str());
 906       }
 907
 908       // all set and done
 909       Complete = true;
 910       if(Debug)
 911          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 912    }
 913 }
 914                                                                         /*}}}*/
 915 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* The package file is added to the queue and a second class is
 918    instantiated to fetch the revision file */
 919 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 920                          string URI,string URIDesc,string ShortDesc,
 921                          HashString ExpectedHash, string comprExt)
 922    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 923 {
 924    if(comprExt.empty() == true)
 925    {
 926       // autoselect the compression method
 927       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 928       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 929          comprExt.append(*t).append(" ");
 930       if (comprExt.empty() == false)
 931          comprExt.erase(comprExt.end()-1);
 932    }
 933    CompressionExtension = comprExt;
 934
 935    Verify = true;
 936
 937    Init(URI, URIDesc, ShortDesc);
 938 }
 939 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 940                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 941    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 942 {
 943    // autoselect the compression method
 944    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 945    CompressionExtension = "";
 946    if (ExpectedHash.empty() == false)
 947    {
 948       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 949          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 950             CompressionExtension.append(*t).append(" ");
 951    }
 952    else
 953    {
 954       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 955          CompressionExtension.append(*t).append(" ");
 956    }
 957    if (CompressionExtension.empty() == false)
 958       CompressionExtension.erase(CompressionExtension.end()-1);
 959
 960    // only verify non-optional targets, see acquire-item.h for a FIXME
 961    // to make this more flexible
 962    if (Target->IsOptional())
 963      Verify = false;
 964    else
 965      Verify = true;
 966
 967    Init(Target->URI, Target->Description, Target->ShortDesc);
 968 }
 969                                                                         /*}}}*/
 970 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 971 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 972    Decompression = false;
 973    Erase = false;
 974
 975    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 976    DestFile += URItoFileName(URI);
 977
 978    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 979    if (comprExt == "uncompressed")
 980       Desc.URI = URI;
 981    else
 982       Desc.URI = URI + '.' + comprExt;
 983
 984    Desc.Description = URIDesc;
 985    Desc.Owner = this;
 986    Desc.ShortDesc = ShortDesc;
 987
 988    QueueURI(Desc);
 989 }
 990                                                                         /*}}}*/
 991 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 992 // ---------------------------------------------------------------------
 993 /* The only header we use is the last-modified header. */
 994 string pkgAcqIndex::Custom600Headers()
 995 {
 996    string Final = _config->FindDir("Dir::State::lists");
 997    Final += URItoFileName(RealURI);
 998    if (_config->FindB("Acquire::GzipIndexes",false))
 999       Final += ".gz";
1000
1001    string msg = "\nIndex-File: true";
1002    // FIXME: this really should use "IndexTarget::IsOptional()" but that
1003    //        seems to be difficult without breaking ABI
1004    if (ShortDesc().find("Translation") != 0)
1005       msg += "\nFail-Ignore: true";
1006    struct stat Buf;
1007    if (stat(Final.c_str(),&Buf) == 0)
1008       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1009
1010    return msg;
1011 }
1012                                                                         /*}}}*/
1013 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1014 {
1015    size_t const nextExt = CompressionExtension.find(' ');
1016    if (nextExt != std::string::npos)
1017    {
1018       CompressionExtension = CompressionExtension.substr(nextExt+1);
1019       Init(RealURI, Desc.Description, Desc.ShortDesc);
1020       return;
1021    }
1022
1023    // on decompression failure, remove bad versions in partial/
1024    if (Decompression && Erase) {
1025       string s = _config->FindDir("Dir::State::lists") + "partial/";
1026       s.append(URItoFileName(RealURI));
1027       unlink(s.c_str());
1028    }
1029
1030    Item::Failed(Message,Cnf);
1031 }
1032                                                                         /*}}}*/
1033 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1034 // ---------------------------------------------------------------------
1035 /* This goes through a number of states.. On the initial fetch the
1036    method could possibly return an alternate filename which points
1037    to the uncompressed version of the file. If this is so the file
1038    is copied into the partial directory. In all other cases the file
1039    is decompressed with a gzip uri. */
1040 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1041                        pkgAcquire::MethodConfig *Cfg)
1042 {
1043    Item::Done(Message,Size,Hash,Cfg);
1044
1045    if (Decompression == true)
1046    {
1047       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1048       {
1049          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1050          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1051       }
1052
1053       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1054       {
1055          RenameOnError(HashSumMismatch);
1056          return;
1057       }
1058
1059       /* Verify the index file for correctness (all indexes must
1060        * have a Package field) (LP: #346386) (Closes: #627642) */
1061       if (Verify == true)
1062       {
1063          FileFd fd(DestFile, FileFd::ReadOnly);
1064          // Only test for correctness if the file is not empty (empty is ok)
1065          if (fd.FileSize() > 0)
1066          {
1067             pkgTagSection sec;
1068             pkgTagFile tag(&fd);
1069
1070             // all our current indexes have a field 'Package' in each section
1071             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1072             {
1073                RenameOnError(InvalidFormat);
1074                return;
1075             }
1076          }
1077       }
1078
1079       // Done, move it into position
1080       string FinalFile = _config->FindDir("Dir::State::lists");
1081       FinalFile += URItoFileName(RealURI);
1082       Rename(DestFile,FinalFile);
1083       chmod(FinalFile.c_str(),0644);
1084
1085       /* We restore the original name to DestFile so that the clean operation
1086          will work OK */
1087       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1088       DestFile += URItoFileName(RealURI);
1089
1090       // Remove the compressed version.
1091       if (Erase == true)
1092          unlink(DestFile.c_str());
1093       return;
1094    }
1095
1096    Erase = false;
1097    Complete = true;
1098
1099    // Handle the unzipd case
1100    string FileName = LookupTag(Message,"Alt-Filename");
1101    if (FileName.empty() == false)
1102    {
1103       // The files timestamp matches
1104       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1105          return;
1106       Decompression = true;
1107       Local = true;
1108       DestFile += ".decomp";
1109       Desc.URI = "copy:" + FileName;
1110       QueueURI(Desc);
1111       Mode = "copy";
1112       return;
1113    }
1114
1115    FileName = LookupTag(Message,"Filename");
1116    if (FileName.empty() == true)
1117    {
1118       Status = StatError;
1119       ErrorText = "Method gave a blank filename";
1120    }
1121
1122    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1123
1124    // The files timestamp matches
1125    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1126        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1127           // Update DestFile for .gz suffix so that the clean operation keeps it
1128           DestFile += ".gz";
1129       return;
1130     }
1131
1132    if (FileName == DestFile)
1133       Erase = true;
1134    else
1135       Local = true;
1136
1137    string decompProg;
1138
1139    // If we enable compressed indexes and already have gzip, keep it
1140    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1141       string FinalFile = _config->FindDir("Dir::State::lists");
1142       FinalFile += URItoFileName(RealURI) + ".gz";
1143       Rename(DestFile,FinalFile);
1144       chmod(FinalFile.c_str(),0644);
1145
1146       // Update DestFile for .gz suffix so that the clean operation keeps it
1147       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1148       DestFile += URItoFileName(RealURI) + ".gz";
1149       return;
1150     }
1151
1152    // get the binary name for your used compression type
1153    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1154    if(decompProg.empty() == false);
1155    else if(compExt == "uncompressed")
1156       decompProg = "copy";
1157    else {
1158       _error->Error("Unsupported extension: %s", compExt.c_str());
1159       return;
1160    }
1161
1162    Decompression = true;
1163    DestFile += ".decomp";
1164    Desc.URI = decompProg + ":" + FileName;
1165    QueueURI(Desc);
1166
1167    // FIXME: this points to a c++ string that goes out of scope
1168    Mode = decompProg.c_str();
1169 }
1170                                                                         /*}}}*/
1171 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1172 // ---------------------------------------------------------------------
1173 /* The Translation file is added to the queue */
1174 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1175                             string URI,string URIDesc,string ShortDesc)
1176   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1177 {
1178 }
1179 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1180                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1181   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1182 {
1183 }
1184                                                                         /*}}}*/
1185 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1186 // ---------------------------------------------------------------------
1187 string pkgAcqIndexTrans::Custom600Headers()
1188 {
1189    string Final = _config->FindDir("Dir::State::lists");
1190    Final += URItoFileName(RealURI);
1191
1192    struct stat Buf;
1193    if (stat(Final.c_str(),&Buf) != 0)
1194       return "\nFail-Ignore: true\nIndex-File: true";
1195    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1196 }
1197                                                                         /*}}}*/
1198 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1199 // ---------------------------------------------------------------------
1200 /* */
1201 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1202 {
1203    size_t const nextExt = CompressionExtension.find(' ');
1204    if (nextExt != std::string::npos)
1205    {
1206       CompressionExtension = CompressionExtension.substr(nextExt+1);
1207       Init(RealURI, Desc.Description, Desc.ShortDesc);
1208       Status = StatIdle;
1209       return;
1210    }
1211
1212    if (Cnf->LocalOnly == true ||
1213        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1214    {
1215       // Ignore this
1216       Status = StatDone;
1217       Complete = false;
1218       Dequeue();
1219       return;
1220    }
1221
1222    Item::Failed(Message,Cnf);
1223 }
1224                                                                         /*}}}*/
1225 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1226                              string URI,string URIDesc,string ShortDesc,
1227                              string MetaIndexURI, string MetaIndexURIDesc,
1228                              string MetaIndexShortDesc,
1229                              const vector<IndexTarget*>* IndexTargets,
1230                              indexRecords* MetaIndexParser) :
1231    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1232    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1233    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1234 {
1235    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1236    DestFile += URItoFileName(URI);
1237
1238    // remove any partial downloaded sig-file in partial/.
1239    // it may confuse proxies and is too small to warrant a
1240    // partial download anyway
1241    unlink(DestFile.c_str());
1242
1243    // Create the item
1244    Desc.Description = URIDesc;
1245    Desc.Owner = this;
1246    Desc.ShortDesc = ShortDesc;
1247    Desc.URI = URI;
1248
1249    string Final = _config->FindDir("Dir::State::lists");
1250    Final += URItoFileName(RealURI);
1251    if (RealFileExists(Final) == true)
1252    {
1253       // File was already in place.  It needs to be re-downloaded/verified
1254       // because Release might have changed, we do give it a different
1255       // name than DestFile because otherwise the http method will
1256       // send If-Range requests and there are too many broken servers
1257       // out there that do not understand them
1258       LastGoodSig = DestFile+".reverify";
1259       Rename(Final,LastGoodSig);
1260    }
1261
1262    QueueURI(Desc);
1263 }
1264                                                                         /*}}}*/
1265 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1266 {
1267    // if the file was never queued undo file-changes done in the constructor
1268    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1269          LastGoodSig.empty() == false)
1270    {
1271       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1272       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1273          Rename(LastGoodSig, Final);
1274    }
1275
1276 }
1277                                                                         /*}}}*/
1278 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1279 // ---------------------------------------------------------------------
1280 /* The only header we use is the last-modified header. */
1281 string pkgAcqMetaSig::Custom600Headers()
1282 {
1283    struct stat Buf;
1284    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1285       return "\nIndex-File: true";
1286
1287    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1288 }
1289
1290 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1291                          pkgAcquire::MethodConfig *Cfg)
1292 {
1293    Item::Done(Message,Size,MD5,Cfg);
1294
1295    string FileName = LookupTag(Message,"Filename");
1296    if (FileName.empty() == true)
1297    {
1298       Status = StatError;
1299       ErrorText = "Method gave a blank filename";
1300       return;
1301    }
1302
1303    if (FileName != DestFile)
1304    {
1305       // We have to copy it into place
1306       Local = true;
1307       Desc.URI = "copy:" + FileName;
1308       QueueURI(Desc);
1309       return;
1310    }
1311
1312    Complete = true;
1313
1314    // put the last known good file back on i-m-s hit (it will
1315    // be re-verified again)
1316    // Else do nothing, we have the new file in DestFile then
1317    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1318       Rename(LastGoodSig, DestFile);
1319
1320    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1321    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1322                        MetaIndexShortDesc,  DestFile, IndexTargets,
1323                        MetaIndexParser);
1324
1325 }
1326                                                                         /*}}}*/
1327 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1328 {
1329    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1330
1331    // if we get a network error we fail gracefully
1332    if(Status == StatTransientNetworkError)
1333    {
1334       Item::Failed(Message,Cnf);
1335       // move the sigfile back on transient network failures
1336       if(FileExists(LastGoodSig))
1337          Rename(LastGoodSig,Final);
1338
1339       // set the status back to , Item::Failed likes to reset it
1340       Status = pkgAcquire::Item::StatTransientNetworkError;
1341       return;
1342    }
1343
1344    // Delete any existing sigfile when the acquire failed
1345    unlink(Final.c_str());
1346
1347    // queue a pkgAcqMetaIndex with no sigfile
1348    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1349                        "", IndexTargets, MetaIndexParser);
1350
1351    if (Cnf->LocalOnly == true ||
1352        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1353    {
1354       // Ignore this
1355       Status = StatDone;
1356       Complete = false;
1357       Dequeue();
1358       return;
1359    }
1360
1361    Item::Failed(Message,Cnf);
1362 }
1363                                                                         /*}}}*/
1364 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1365                                  string URI,string URIDesc,string ShortDesc,
1366                                  string SigFile,
1367                                  const vector<struct IndexTarget*>* IndexTargets,
1368                                  indexRecords* MetaIndexParser) :
1369    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1370    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1371 {
1372    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1373    DestFile += URItoFileName(URI);
1374
1375    // Create the item
1376    Desc.Description = URIDesc;
1377    Desc.Owner = this;
1378    Desc.ShortDesc = ShortDesc;
1379    Desc.URI = URI;
1380
1381    QueueURI(Desc);
1382 }
1383                                                                         /*}}}*/
1384 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1385 // ---------------------------------------------------------------------
1386 /* The only header we use is the last-modified header. */
1387 string pkgAcqMetaIndex::Custom600Headers()
1388 {
1389    string Final = _config->FindDir("Dir::State::lists");
1390    Final += URItoFileName(RealURI);
1391
1392    struct stat Buf;
1393    if (stat(Final.c_str(),&Buf) != 0)
1394       return "\nIndex-File: true";
1395
1396    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1397 }
1398                                                                         /*}}}*/
1399 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1400                            pkgAcquire::MethodConfig *Cfg)
1401 {
1402    Item::Done(Message,Size,Hash,Cfg);
1403
1404    // MetaIndexes are done in two passes: one to download the
1405    // metaindex with an appropriate method, and a second to verify it
1406    // with the gpgv method
1407
1408    if (AuthPass == true)
1409    {
1410       AuthDone(Message);
1411
1412       // all cool, move Release file into place
1413       Complete = true;
1414    }
1415    else
1416    {
1417       RetrievalDone(Message);
1418       if (!Complete)
1419          // Still more retrieving to do
1420          return;
1421
1422       if (SigFile == "")
1423       {
1424          // There was no signature file, so we are finished.  Download
1425          // the indexes and do only hashsum verification if possible
1426          MetaIndexParser->Load(DestFile);
1427          QueueIndexes(false);
1428       }
1429       else
1430       {
1431          // There was a signature file, so pass it to gpgv for
1432          // verification
1433
1434          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1435             std::cerr << "Metaindex acquired, queueing gpg verification ("
1436                       << SigFile << "," << DestFile << ")\n";
1437          AuthPass = true;
1438          Desc.URI = "gpgv:" + SigFile;
1439          QueueURI(Desc);
1440          Mode = "gpgv";
1441          return;
1442       }
1443    }
1444
1445    if (Complete == true)
1446    {
1447       string FinalFile = _config->FindDir("Dir::State::lists");
1448       FinalFile += URItoFileName(RealURI);
1449       if (SigFile == DestFile)
1450          SigFile = FinalFile;
1451       Rename(DestFile,FinalFile);
1452       chmod(FinalFile.c_str(),0644);
1453       DestFile = FinalFile;
1454    }
1455 }
1456                                                                         /*}}}*/
1457 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1458 {
1459    // We have just finished downloading a Release file (it is not
1460    // verified yet)
1461
1462    string FileName = LookupTag(Message,"Filename");
1463    if (FileName.empty() == true)
1464    {
1465       Status = StatError;
1466       ErrorText = "Method gave a blank filename";
1467       return;
1468    }
1469
1470    if (FileName != DestFile)
1471    {
1472       Local = true;
1473       Desc.URI = "copy:" + FileName;
1474       QueueURI(Desc);
1475       return;
1476    }
1477
1478    // make sure to verify against the right file on I-M-S hit
1479    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1480    if(IMSHit)
1481    {
1482       string FinalFile = _config->FindDir("Dir::State::lists");
1483       FinalFile += URItoFileName(RealURI);
1484       if (SigFile == DestFile)
1485       {
1486          SigFile = FinalFile;
1487          // constructor of pkgAcqMetaClearSig moved it out of the way,
1488          // now move it back in on IMS hit for the 'old' file
1489          string const OldClearSig = DestFile + ".reverify";
1490          if (RealFileExists(OldClearSig) == true)
1491             Rename(OldClearSig, FinalFile);
1492       }
1493       DestFile = FinalFile;
1494    }
1495    Complete = true;
1496 }
1497                                                                         /*}}}*/
1498 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1499 {
1500    // At this point, the gpgv method has succeeded, so there is a
1501    // valid signature from a key in the trusted keyring.  We
1502    // perform additional verification of its contents, and use them
1503    // to verify the indexes we are about to download
1504
1505    if (!MetaIndexParser->Load(DestFile))
1506    {
1507       Status = StatAuthError;
1508       ErrorText = MetaIndexParser->ErrorText;
1509       return;
1510    }
1511
1512    if (!VerifyVendor(Message))
1513    {
1514       return;
1515    }
1516
1517    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1518       std::cerr << "Signature verification succeeded: "
1519                 << DestFile << std::endl;
1520
1521    // Download further indexes with verification
1522    QueueIndexes(true);
1523
1524    // is it a clearsigned MetaIndex file?
1525    if (DestFile == SigFile)
1526       return;
1527
1528    // Done, move signature file into position
1529    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1530       URItoFileName(RealURI) + ".gpg";
1531    Rename(SigFile,VerifiedSigFile);
1532    chmod(VerifiedSigFile.c_str(),0644);
1533 }
1534                                                                         /*}}}*/
1535 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1536 {
1537 #if 0
1538    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1539     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1540    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1541    {
1542       Status = StatError;
1543       ErrorText = MetaIndexParser->ErrorText;
1544       return;
1545    }
1546 #endif
1547    bool transInRelease = false;
1548    {
1549       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1550       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1551          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1552          if (k->find("Translation-") != std::string::npos)
1553          {
1554             transInRelease = true;
1555             break;
1556          }
1557    }
1558
1559    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1560         Target != IndexTargets->end();
1561         ++Target)
1562    {
1563       HashString ExpectedIndexHash;
1564       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1565       bool compressedAvailable = false;
1566       if (Record == NULL)
1567       {
1568          if ((*Target)->IsOptional() == true)
1569          {
1570             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1571             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1572                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1573                {
1574                   compressedAvailable = true;
1575                   break;
1576                }
1577          }
1578          else if (verify == true)
1579          {
1580             Status = StatAuthError;
1581             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1582             return;
1583          }
1584       }
1585       else
1586       {
1587          ExpectedIndexHash = Record->Hash;
1588          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1589          {
1590             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1591             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1592             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1593          }
1594          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1595          {
1596             Status = StatAuthError;
1597             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1598             return;
1599          }
1600       }
1601
1602       if ((*Target)->IsOptional() == true)
1603       {
1604          if ((*Target)->IsSubIndex() == true)
1605             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1606                                 (*Target)->ShortDesc, ExpectedIndexHash);
1607          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1608          {
1609             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1610                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1611                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1612                                    (*Target)->ShortDesc, ExpectedIndexHash);
1613             else
1614                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1615          }
1616          continue;
1617       }
1618
1619       /* Queue Packages file (either diff or full packages files, depending
1620          on the users option) - we also check if the PDiff Index file is listed
1621          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1622          instead, but passing the required info to it is to much hassle */
1623       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1624           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1625          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1626                              (*Target)->ShortDesc, ExpectedIndexHash);
1627       else
1628          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1629    }
1630 }
1631                                                                         /*}}}*/
1632 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1633 {
1634    string::size_type pos;
1635
1636    // check for missing sigs (that where not fatal because otherwise we had
1637    // bombed earlier)
1638    string missingkeys;
1639    string msg = _("There is no public key available for the "
1640                   "following key IDs:\n");
1641    pos = Message.find("NO_PUBKEY ");
1642    if (pos != std::string::npos)
1643    {
1644       string::size_type start = pos+strlen("NO_PUBKEY ");
1645       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1646       missingkeys += (Fingerprint);
1647    }
1648    if(!missingkeys.empty())
1649       _error->Warning("%s", (msg + missingkeys).c_str());
1650
1651    string Transformed = MetaIndexParser->GetExpectedDist();
1652
1653    if (Transformed == "../project/experimental")
1654    {
1655       Transformed = "experimental";
1656    }
1657
1658    pos = Transformed.rfind('/');
1659    if (pos != string::npos)
1660    {
1661       Transformed = Transformed.substr(0, pos);
1662    }
1663
1664    if (Transformed == ".")
1665    {
1666       Transformed = "";
1667    }
1668
1669    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1670        MetaIndexParser->GetValidUntil() > 0) {
1671       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1672       if (invalid_since > 0)
1673          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1674          // the time since then the file is invalid - formated in the same way as in
1675          // the download progress display (e.g. 7d 3h 42min 1s)
1676          return _error->Error(
1677             _("Release file for %s is expired (invalid since %s). "
1678               "Updates for this repository will not be applied."),
1679             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1680    }
1681
1682    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1683    {
1684       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1685       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1686       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1687    }
1688
1689    if (MetaIndexParser->CheckDist(Transformed) == false)
1690    {
1691       // This might become fatal one day
1692 //       Status = StatAuthError;
1693 //       ErrorText = "Conflicting distribution; expected "
1694 //          + MetaIndexParser->GetExpectedDist() + " but got "
1695 //          + MetaIndexParser->GetDist();
1696 //       return false;
1697       if (!Transformed.empty())
1698       {
1699          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1700                          Desc.Description.c_str(),
1701                          Transformed.c_str(),
1702                          MetaIndexParser->GetDist().c_str());
1703       }
1704    }
1705
1706    return true;
1707 }
1708                                                                         /*}}}*/
1709 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1710 // ---------------------------------------------------------------------
1711 /* */
1712 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1713 {
1714    if (AuthPass == true)
1715    {
1716       // gpgv method failed, if we have a good signature
1717       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1718       if (DestFile != SigFile)
1719          LastGoodSigFile.append(".gpg");
1720       LastGoodSigFile.append(".reverify");
1721
1722       if(FileExists(LastGoodSigFile))
1723       {
1724          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1725          if (DestFile != SigFile)
1726             VerifiedSigFile.append(".gpg");
1727          Rename(LastGoodSigFile, VerifiedSigFile);
1728          Status = StatTransientNetworkError;
1729          _error->Warning(_("An error occurred during the signature "
1730                            "verification. The repository is not updated "
1731                            "and the previous index files will be used. "
1732                            "GPG error: %s: %s\n"),
1733                          Desc.Description.c_str(),
1734                          LookupTag(Message,"Message").c_str());
1735          RunScripts("APT::Update::Auth-Failure");
1736          return;
1737       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1738          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1739          _error->Error(_("GPG error: %s: %s"),
1740                          Desc.Description.c_str(),
1741                          LookupTag(Message,"Message").c_str());
1742          return;
1743       } else {
1744          _error->Warning(_("GPG error: %s: %s"),
1745                          Desc.Description.c_str(),
1746                          LookupTag(Message,"Message").c_str());
1747       }
1748       // gpgv method failed
1749       ReportMirrorFailure("GPGFailure");
1750    }
1751
1752    /* Always move the meta index, even if gpgv failed. This ensures
1753     * that PackageFile objects are correctly filled in */
1754    if (FileExists(DestFile)) {
1755       string FinalFile = _config->FindDir("Dir::State::lists");
1756       FinalFile += URItoFileName(RealURI);
1757       /* InRelease files become Release files, otherwise
1758        * they would be considered as trusted later on */
1759       if (SigFile == DestFile) {
1760          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1761                                        "Release");
1762          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1763                                        "Release");
1764          SigFile = FinalFile;
1765       }
1766       Rename(DestFile,FinalFile);
1767       chmod(FinalFile.c_str(),0644);
1768
1769       DestFile = FinalFile;
1770    }
1771
1772    // No Release file was present, or verification failed, so fall
1773    // back to queueing Packages files without verification
1774    QueueIndexes(false);
1775 }
1776                                                                         /*}}}*/
1777 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1778                 string const &URI, string const &URIDesc, string const &ShortDesc,
1779                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1780                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1781                 const vector<struct IndexTarget*>* IndexTargets,
1782                 indexRecords* MetaIndexParser) :
1783         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1784         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1785         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1786 {
1787    SigFile = DestFile;
1788
1789    // keep the old InRelease around in case of transistent network errors
1790    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1791    if (RealFileExists(Final) == true)
1792    {
1793       string const LastGoodSig = DestFile + ".reverify";
1794       Rename(Final,LastGoodSig);
1795    }
1796 }
1797                                                                         /*}}}*/
1798 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1799 {
1800    // if the file was never queued undo file-changes done in the constructor
1801    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1802    {
1803       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1804       string const LastGoodSig = DestFile + ".reverify";
1805       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1806          Rename(LastGoodSig, Final);
1807    }
1808 }
1809                                                                         /*}}}*/
1810 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1811 // ---------------------------------------------------------------------
1812 // FIXME: this can go away once the InRelease file is used widely
1813 string pkgAcqMetaClearSig::Custom600Headers()
1814 {
1815    string Final = _config->FindDir("Dir::State::lists");
1816    Final += URItoFileName(RealURI);
1817
1818    struct stat Buf;
1819    if (stat(Final.c_str(),&Buf) != 0)
1820    {
1821       Final = DestFile + ".reverify";
1822       if (stat(Final.c_str(),&Buf) != 0)
1823          return "\nIndex-File: true\nFail-Ignore: true\n";
1824    }
1825
1826    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1827 }
1828                                                                         /*}}}*/
1829 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1830 {
1831    if (AuthPass == false)
1832    {
1833       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1834       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1835       string FinalFile = _config->FindDir("Dir::State::lists");
1836       FinalFile.append(URItoFileName(RealURI));
1837       if (FileExists(FinalFile))
1838          unlink(FinalFile.c_str());
1839
1840       new pkgAcqMetaSig(Owner,
1841                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1842                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1843                         IndexTargets, MetaIndexParser);
1844       if (Cnf->LocalOnly == true ||
1845           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1846          Dequeue();
1847    }
1848    else
1849       pkgAcqMetaIndex::Failed(Message, Cnf);
1850 }
1851                                                                         /*}}}*/
1852 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1853 // ---------------------------------------------------------------------
1854 /* This just sets up the initial fetch environment and queues the first
1855    possibilitiy */
1856 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1857                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1858                              string &StoreFilename) :
1859                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1860                StoreFilename(StoreFilename), Vf(Version.FileList()),
1861                Trusted(false)
1862 {
1863    Retries = _config->FindI("Acquire::Retries",0);
1864
1865    if (Version.Arch() == 0)
1866    {
1867       _error->Error(_("I wasn't able to locate a file for the %s package. "
1868                       "This might mean you need to manually fix this package. "
1869                       "(due to missing arch)"),
1870                     Version.ParentPkg().FullName().c_str());
1871       return;
1872    }
1873
1874    /* We need to find a filename to determine the extension. We make the
1875       assumption here that all the available sources for this version share
1876       the same extension.. */
1877    // Skip not source sources, they do not have file fields.
1878    for (; Vf.end() == false; ++Vf)
1879    {
1880       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1881          continue;
1882       break;
1883    }
1884
1885    // Does not really matter here.. we are going to fail out below
1886    if (Vf.end() != true)
1887    {
1888       // If this fails to get a file name we will bomb out below.
1889       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1890       if (_error->PendingError() == true)
1891          return;
1892
1893       // Generate the final file name as: package_version_arch.foo
1894       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1895                       QuoteString(Version.VerStr(),"_:") + '_' +
1896                       QuoteString(Version.Arch(),"_:.") +
1897                       "." + flExtension(Parse.FileName());
1898    }
1899
1900    // check if we have one trusted source for the package. if so, switch
1901    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1902    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1903    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1904    bool seenUntrusted = false;
1905    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1906    {
1907       pkgIndexFile *Index;
1908       if (Sources->FindIndex(i.File(),Index) == false)
1909          continue;
1910
1911       if (debugAuth == true)
1912          std::cerr << "Checking index: " << Index->Describe()
1913                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1914
1915       if (Index->IsTrusted() == true)
1916       {
1917          Trusted = true;
1918          if (allowUnauth == false)
1919             break;
1920       }
1921       else
1922          seenUntrusted = true;
1923    }
1924
1925    // "allow-unauthenticated" restores apts old fetching behaviour
1926    // that means that e.g. unauthenticated file:// uris are higher
1927    // priority than authenticated http:// uris
1928    if (allowUnauth == true && seenUntrusted == true)
1929       Trusted = false;
1930
1931    // Select a source
1932    if (QueueNext() == false && _error->PendingError() == false)
1933       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1934                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1935 }
1936                                                                         /*}}}*/
1937 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1938 // ---------------------------------------------------------------------
1939 /* This queues the next available file version for download. It checks if
1940    the archive is already available in the cache and stashs the MD5 for
1941    checking later. */
1942 bool pkgAcqArchive::QueueNext()
1943 {
1944    string const ForceHash = _config->Find("Acquire::ForceHash");
1945    for (; Vf.end() == false; ++Vf)
1946    {
1947       // Ignore not source sources
1948       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1949          continue;
1950
1951       // Try to cross match against the source list
1952       pkgIndexFile *Index;
1953       if (Sources->FindIndex(Vf.File(),Index) == false)
1954             continue;
1955
1956       // only try to get a trusted package from another source if that source
1957       // is also trusted
1958       if(Trusted && !Index->IsTrusted())
1959          continue;
1960
1961       // Grab the text package record
1962       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1963       if (_error->PendingError() == true)
1964          return false;
1965
1966       string PkgFile = Parse.FileName();
1967       if (ForceHash.empty() == false)
1968       {
1969          if(stringcasecmp(ForceHash, "sha512") == 0)
1970             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
1971          else if(stringcasecmp(ForceHash, "sha256") == 0)
1972             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1973          else if (stringcasecmp(ForceHash, "sha1") == 0)
1974             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1975          else
1976             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1977       }
1978       else
1979       {
1980          string Hash;
1981          if ((Hash = Parse.SHA512Hash()).empty() == false)
1982             ExpectedHash = HashString("SHA512", Hash);
1983          else if ((Hash = Parse.SHA256Hash()).empty() == false)
1984             ExpectedHash = HashString("SHA256", Hash);
1985          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1986             ExpectedHash = HashString("SHA1", Hash);
1987          else
1988             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1989       }
1990       if (PkgFile.empty() == true)
1991          return _error->Error(_("The package index files are corrupted. No Filename: "
1992                               "field for package %s."),
1993                               Version.ParentPkg().Name());
1994
1995       Desc.URI = Index->ArchiveURI(PkgFile);
1996       Desc.Description = Index->ArchiveInfo(Version);
1997       Desc.Owner = this;
1998       Desc.ShortDesc = Version.ParentPkg().FullName(true);
1999
2000       // See if we already have the file. (Legacy filenames)
2001       FileSize = Version->Size;
2002       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2003       struct stat Buf;
2004       if (stat(FinalFile.c_str(),&Buf) == 0)
2005       {
2006          // Make sure the size matches
2007          if ((unsigned long long)Buf.st_size == Version->Size)
2008          {
2009             Complete = true;
2010             Local = true;
2011             Status = StatDone;
2012             StoreFilename = DestFile = FinalFile;
2013             return true;
2014          }
2015
2016          /* Hmm, we have a file and its size does not match, this means it is
2017             an old style mismatched arch */
2018          unlink(FinalFile.c_str());
2019       }
2020
2021       // Check it again using the new style output filenames
2022       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2023       if (stat(FinalFile.c_str(),&Buf) == 0)
2024       {
2025          // Make sure the size matches
2026          if ((unsigned long long)Buf.st_size == Version->Size)
2027          {
2028             Complete = true;
2029             Local = true;
2030             Status = StatDone;
2031             StoreFilename = DestFile = FinalFile;
2032             return true;
2033          }
2034
2035          /* Hmm, we have a file and its size does not match, this shouldn't
2036             happen.. */
2037          unlink(FinalFile.c_str());
2038       }
2039
2040       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2041
2042       // Check the destination file
2043       if (stat(DestFile.c_str(),&Buf) == 0)
2044       {
2045          // Hmm, the partial file is too big, erase it
2046          if ((unsigned long long)Buf.st_size > Version->Size)
2047             unlink(DestFile.c_str());
2048          else
2049             PartialSize = Buf.st_size;
2050       }
2051
2052       // Disables download of archives - useful if no real installation follows,
2053       // e.g. if we are just interested in proposed installation order
2054       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2055       {
2056          Complete = true;
2057          Local = true;
2058          Status = StatDone;
2059          StoreFilename = DestFile = FinalFile;
2060          return true;
2061       }
2062
2063       // Create the item
2064       Local = false;
2065       QueueURI(Desc);
2066
2067       ++Vf;
2068       return true;
2069    }
2070    return false;
2071 }
2072                                                                         /*}}}*/
2073 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2074 // ---------------------------------------------------------------------
2075 /* */
2076 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2077                          pkgAcquire::MethodConfig *Cfg)
2078 {
2079    Item::Done(Message,Size,CalcHash,Cfg);
2080
2081    // Check the size
2082    if (Size != Version->Size)
2083    {
2084       RenameOnError(SizeMismatch);
2085       return;
2086    }
2087
2088    // Check the hash
2089    if(ExpectedHash.toStr() != CalcHash)
2090    {
2091       RenameOnError(HashSumMismatch);
2092       return;
2093    }
2094
2095    // Grab the output filename
2096    string FileName = LookupTag(Message,"Filename");
2097    if (FileName.empty() == true)
2098    {
2099       Status = StatError;
2100       ErrorText = "Method gave a blank filename";
2101       return;
2102    }
2103
2104    Complete = true;
2105
2106    // Reference filename
2107    if (FileName != DestFile)
2108    {
2109       StoreFilename = DestFile = FileName;
2110       Local = true;
2111       return;
2112    }
2113
2114    // Done, move it into position
2115    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2116    FinalFile += flNotDir(StoreFilename);
2117    Rename(DestFile,FinalFile);
2118
2119    StoreFilename = DestFile = FinalFile;
2120    Complete = true;
2121 }
2122                                                                         /*}}}*/
2123 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2124 // ---------------------------------------------------------------------
2125 /* Here we try other sources */
2126 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2127 {
2128    ErrorText = LookupTag(Message,"Message");
2129
2130    /* We don't really want to retry on failed media swaps, this prevents
2131       that. An interesting observation is that permanent failures are not
2132       recorded. */
2133    if (Cnf->Removable == true &&
2134        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2135    {
2136       // Vf = Version.FileList();
2137       while (Vf.end() == false) ++Vf;
2138       StoreFilename = string();
2139       Item::Failed(Message,Cnf);
2140       return;
2141    }
2142
2143    if (QueueNext() == false)
2144    {
2145       // This is the retry counter
2146       if (Retries != 0 &&
2147           Cnf->LocalOnly == false &&
2148           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2149       {
2150          Retries--;
2151          Vf = Version.FileList();
2152          if (QueueNext() == true)
2153             return;
2154       }
2155
2156       StoreFilename = string();
2157       Item::Failed(Message,Cnf);
2158    }
2159 }
2160                                                                         /*}}}*/
2161 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2162 // ---------------------------------------------------------------------
2163 APT_PURE bool pkgAcqArchive::IsTrusted()
2164 {
2165    return Trusted;
2166 }
2167                                                                         /*}}}*/
2168 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2169 // ---------------------------------------------------------------------
2170 /* */
2171 void pkgAcqArchive::Finished()
2172 {
2173    if (Status == pkgAcquire::Item::StatDone &&
2174        Complete == true)
2175       return;
2176    StoreFilename = string();
2177 }
2178                                                                         /*}}}*/
2179 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2180 // ---------------------------------------------------------------------
2181 /* The file is added to the queue */
2182 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2183                        unsigned long long Size,string Dsc,string ShortDesc,
2184                        const string &DestDir, const string &DestFilename,
2185                        bool IsIndexFile) :
2186                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2187 {
2188    Retries = _config->FindI("Acquire::Retries",0);
2189
2190    if(!DestFilename.empty())
2191       DestFile = DestFilename;
2192    else if(!DestDir.empty())
2193       DestFile = DestDir + "/" + flNotDir(URI);
2194    else
2195       DestFile = flNotDir(URI);
2196
2197    // Create the item
2198    Desc.URI = URI;
2199    Desc.Description = Dsc;
2200    Desc.Owner = this;
2201
2202    // Set the short description to the archive component
2203    Desc.ShortDesc = ShortDesc;
2204
2205    // Get the transfer sizes
2206    FileSize = Size;
2207    struct stat Buf;
2208    if (stat(DestFile.c_str(),&Buf) == 0)
2209    {
2210       // Hmm, the partial file is too big, erase it
2211       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2212          unlink(DestFile.c_str());
2213       else
2214          PartialSize = Buf.st_size;
2215    }
2216
2217    QueueURI(Desc);
2218 }
2219                                                                         /*}}}*/
2220 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2221 // ---------------------------------------------------------------------
2222 /* */
2223 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2224                       pkgAcquire::MethodConfig *Cnf)
2225 {
2226    Item::Done(Message,Size,CalcHash,Cnf);
2227
2228    // Check the hash
2229    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2230    {
2231       RenameOnError(HashSumMismatch);
2232       return;
2233    }
2234
2235    string FileName = LookupTag(Message,"Filename");
2236    if (FileName.empty() == true)
2237    {
2238       Status = StatError;
2239       ErrorText = "Method gave a blank filename";
2240       return;
2241    }
2242
2243    Complete = true;
2244
2245    // The files timestamp matches
2246    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2247       return;
2248
2249    // We have to copy it into place
2250    if (FileName != DestFile)
2251    {
2252       Local = true;
2253       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2254           Cnf->Removable == true)
2255       {
2256          Desc.URI = "copy:" + FileName;
2257          QueueURI(Desc);
2258          return;
2259       }
2260
2261       // Erase the file if it is a symlink so we can overwrite it
2262       struct stat St;
2263       if (lstat(DestFile.c_str(),&St) == 0)
2264       {
2265          if (S_ISLNK(St.st_mode) != 0)
2266             unlink(DestFile.c_str());
2267       }
2268
2269       // Symlink the file
2270       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2271       {
2272          ErrorText = "Link to " + DestFile + " failure ";
2273          Status = StatError;
2274          Complete = false;
2275       }
2276    }
2277 }
2278                                                                         /*}}}*/
2279 // AcqFile::Failed - Failure handler                                    /*{{{*/
2280 // ---------------------------------------------------------------------
2281 /* Here we try other sources */
2282 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2283 {
2284    ErrorText = LookupTag(Message,"Message");
2285
2286    // This is the retry counter
2287    if (Retries != 0 &&
2288        Cnf->LocalOnly == false &&
2289        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2290    {
2291       Retries--;
2292       QueueURI(Desc);
2293       return;
2294    }
2295
2296    Item::Failed(Message,Cnf);
2297 }
2298                                                                         /*}}}*/
2299 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2300 // ---------------------------------------------------------------------
2301 /* The only header we use is the last-modified header. */
2302 string pkgAcqFile::Custom600Headers()
2303 {
2304    if (IsIndexFile)
2305       return "\nIndex-File: true";
2306    return "";
2307 }
2308                                                                         /*}}}*/