apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::Item::Item - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  57                        PartialSize(0), Mode(0), ID(0), Complete(false),
  58                        Local(false), QueueCounter(0)
  59 {
  60    Owner->Add(this);
  61    Status = StatIdle;
  62 }
  63                                                                         /*}}}*/
  64 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 pkgAcquire::Item::~Item()
  68 {
  69    Owner->Remove(this);
  70 }
  71                                                                         /*}}}*/
  72 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  73 // ---------------------------------------------------------------------
  74 /* We return to an idle state if there are still other queues that could
  75    fetch this object */
  76 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  77 {
  78    Status = StatIdle;
  79    ErrorText = LookupTag(Message,"Message");
  80    UsedMirror =  LookupTag(Message,"UsedMirror");
  81    if (QueueCounter <= 1)
  82    {
  83       /* This indicates that the file is not available right now but might
  84          be sometime later. If we do a retry cycle then this should be
  85          retried [CDROMs] */
  86       if (Cnf->LocalOnly == true &&
  87           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  88       {
  89          Status = StatIdle;
  90          Dequeue();
  91          return;
  92       }
  93
  94       Status = StatError;
  95       Dequeue();
  96    }
  97
  98    // report mirror failure back to LP if we actually use a mirror
  99    string FailReason = LookupTag(Message, "FailReason");
 100    if(FailReason.size() != 0)
 101       ReportMirrorFailure(FailReason);
 102    else
 103       ReportMirrorFailure(ErrorText);
 104 }
 105                                                                         /*}}}*/
 106 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* Stash status and the file size. Note that setting Complete means
 109    sub-phases of the acquire process such as decompresion are operating */
 110 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 111 {
 112    Status = StatFetching;
 113    if (FileSize == 0 && Complete == false)
 114       FileSize = Size;
 115 }
 116                                                                         /*}}}*/
 117 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/,
 121                             pkgAcquire::MethodConfig * /*Cnf*/)
 122 {
 123    // We just downloaded something..
 124    string FileName = LookupTag(Message,"Filename");
 125    UsedMirror =  LookupTag(Message,"UsedMirror");
 126    if (Complete == false && !Local && FileName == DestFile)
 127    {
 128       if (Owner->Log != 0)
 129          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 130    }
 131
 132    if (FileSize == 0)
 133       FileSize= Size;
 134    Status = StatDone;
 135    ErrorText = string();
 136    Owner->Dequeue(this);
 137 }
 138                                                                         /*}}}*/
 139 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This helper function is used by a lot of item methods as their final
 142    step */
 143 void pkgAcquire::Item::Rename(string From,string To)
 144 {
 145    if (rename(From.c_str(),To.c_str()) != 0)
 146    {
 147       char S[300];
 148       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 149               From.c_str(),To.c_str());
 150       Status = StatError;
 151       ErrorText = S;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 156 {
 157    if(FileExists(DestFile))
 158       Rename(DestFile, DestFile + ".FAILED");
 159
 160    switch (error)
 161    {
 162       case HashSumMismatch:
 163          ErrorText = _("Hash Sum mismatch");
 164          Status = StatAuthError;
 165          ReportMirrorFailure("HashChecksumFailure");
 166          break;
 167       case SizeMismatch:
 168          ErrorText = _("Size mismatch");
 169          Status = StatAuthError;
 170          ReportMirrorFailure("SizeFailure");
 171          break;
 172       case InvalidFormat:
 173          ErrorText = _("Invalid file format");
 174          Status = StatError;
 175          // do not report as usually its not the mirrors fault, but Portal/Proxy
 176          break;
 177    }
 178    return false;
 179 }
 180                                                                         /*}}}*/
 181 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 182 // ---------------------------------------------------------------------
 183 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 184 {
 185    // we only act if a mirror was used at all
 186    if(UsedMirror.empty())
 187       return;
 188 #if 0
 189    std::cerr << "\nReportMirrorFailure: "
 190              << UsedMirror
 191              << " Uri: " << DescURI()
 192              << " FailCode: "
 193              << FailCode << std::endl;
 194 #endif
 195    const char *Args[40];
 196    unsigned int i = 0;
 197    string report = _config->Find("Methods::Mirror::ProblemReporting",
 198                                  "/usr/lib/apt/apt-report-mirror-failure");
 199    if(!FileExists(report))
 200       return;
 201    Args[i++] = report.c_str();
 202    Args[i++] = UsedMirror.c_str();
 203    Args[i++] = DescURI().c_str();
 204    Args[i++] = FailCode.c_str();
 205    Args[i++] = NULL;
 206    pid_t pid = ExecFork();
 207    if(pid < 0)
 208    {
 209       _error->Error("ReportMirrorFailure Fork failed");
 210       return;
 211    }
 212    else if(pid == 0)
 213    {
 214       execvp(Args[0], (char**)Args);
 215       std::cerr << "Could not exec " << Args[0] << std::endl;
 216       _exit(100);
 217    }
 218    if(!ExecWait(pid, "report-mirror-failure"))
 219    {
 220       _error->Warning("Couldn't report problem to '%s'",
 221                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 222    }
 223 }
 224                                                                         /*}}}*/
 225 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 226 // ---------------------------------------------------------------------
 227 /* Get a sub-index file based on checksums from a 'master' file and
 228    possibly query additional files */
 229 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 230                                  string const &URIDesc, string const &ShortDesc,
 231                                  HashString const &ExpectedHash)
 232    : Item(Owner), ExpectedHash(ExpectedHash)
 233 {
 234    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 235    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 236
 237    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 238    DestFile += URItoFileName(URI);
 239
 240    Desc.URI = URI;
 241    Desc.Description = URIDesc;
 242    Desc.Owner = this;
 243    Desc.ShortDesc = ShortDesc;
 244
 245    QueueURI(Desc);
 246
 247    if(Debug)
 248       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 249 }
 250                                                                         /*}}}*/
 251 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 252 // ---------------------------------------------------------------------
 253 /* The only header we use is the last-modified header. */
 254 string pkgAcqSubIndex::Custom600Headers()
 255 {
 256    string Final = _config->FindDir("Dir::State::lists");
 257    Final += URItoFileName(Desc.URI);
 258
 259    struct stat Buf;
 260    if (stat(Final.c_str(),&Buf) != 0)
 261       return "\nIndex-File: true\nFail-Ignore: true\n";
 262    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 263 }
 264                                                                         /*}}}*/
 265 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 266 {
 267    if(Debug)
 268       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 269
 270    Complete = false;
 271    Status = StatDone;
 272    Dequeue();
 273
 274    // No good Index is provided
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 278                            pkgAcquire::MethodConfig *Cnf)
 279 {
 280    if(Debug)
 281       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 282
 283    string FileName = LookupTag(Message,"Filename");
 284    if (FileName.empty() == true)
 285    {
 286       Status = StatError;
 287       ErrorText = "Method gave a blank filename";
 288       return;
 289    }
 290
 291    if (FileName != DestFile)
 292    {
 293       Local = true;
 294       Desc.URI = "copy:" + FileName;
 295       QueueURI(Desc);
 296       return;
 297    }
 298
 299    Item::Done(Message,Size,Md5Hash,Cnf);
 300
 301    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 302
 303    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 304    indexRecords SubIndexParser;
 305    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 306       Status = StatError;
 307       ErrorText = SubIndexParser.ErrorText;
 308       return;
 309    }
 310
 311    // success in downloading the index
 312    // rename the index
 313    if(Debug)
 314       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 315    Rename(DestFile,FinalFile);
 316    chmod(FinalFile.c_str(),0644);
 317    DestFile = FinalFile;
 318
 319    if(ParseIndex(DestFile) == false)
 320       return Failed("", NULL);
 321
 322    Complete = true;
 323    Status = StatDone;
 324    Dequeue();
 325    return;
 326 }
 327                                                                         /*}}}*/
 328 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 329 {
 330    indexRecords SubIndexParser;
 331    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 332       return false;
 333    // so something with the downloaded index
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* Get the DiffIndex file first and see if there are patches available
 340  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 341  * patches. If anything goes wrong in that process, it will fall back to
 342  * the original packages file
 343  */
 344 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 345                                  string URI,string URIDesc,string ShortDesc,
 346                                  HashString ExpectedHash)
 347    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 348      Description(URIDesc)
 349 {
 350
 351    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 352
 353    Desc.Description = URIDesc + "/DiffIndex";
 354    Desc.Owner = this;
 355    Desc.ShortDesc = ShortDesc;
 356    Desc.URI = URI + ".diff/Index";
 357
 358    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 359    DestFile += URItoFileName(Desc.URI);
 360
 361    if(Debug)
 362       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 363
 364    // look for the current package file
 365    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 366    CurrentPackagesFile += URItoFileName(RealURI);
 367
 368    // FIXME: this file:/ check is a hack to prevent fetching
 369    //        from local sources. this is really silly, and
 370    //        should be fixed cleanly as soon as possible
 371    if(!FileExists(CurrentPackagesFile) ||
 372       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 373    {
 374       // we don't have a pkg file or we don't want to queue
 375       if(Debug)
 376          std::clog << "No index file, local or canceld by user" << std::endl;
 377       Failed("", NULL);
 378       return;
 379    }
 380
 381    if(Debug)
 382       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 383          << CurrentPackagesFile << std::endl;
 384
 385    QueueURI(Desc);
 386
 387 }
 388                                                                         /*}}}*/
 389 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* The only header we use is the last-modified header. */
 392 string pkgAcqDiffIndex::Custom600Headers()
 393 {
 394    string Final = _config->FindDir("Dir::State::lists");
 395    Final += URItoFileName(Desc.URI);
 396
 397    if(Debug)
 398       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 399
 400    struct stat Buf;
 401    if (stat(Final.c_str(),&Buf) != 0)
 402       return "\nIndex-File: true";
 403
 404    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 405 }
 406                                                                         /*}}}*/
 407 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 408 {
 409    if(Debug)
 410       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 411          << std::endl;
 412
 413    pkgTagSection Tags;
 414    string ServerSha1;
 415    vector<DiffInfo> available_patches;
 416
 417    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 418    pkgTagFile TF(&Fd);
 419    if (_error->PendingError() == true)
 420       return false;
 421
 422    if(TF.Step(Tags) == true)
 423    {
 424       bool found = false;
 425       DiffInfo d;
 426       string size;
 427
 428       string const tmp = Tags.FindS("SHA1-Current");
 429       std::stringstream ss(tmp);
 430       ss >> ServerSha1 >> size;
 431       unsigned long const ServerSize = atol(size.c_str());
 432
 433       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 434       SHA1Summation SHA1;
 435       SHA1.AddFD(fd);
 436       string const local_sha1 = SHA1.Result();
 437
 438       if(local_sha1 == ServerSha1)
 439       {
 440          // we have the same sha1 as the server so we are done here
 441          if(Debug)
 442             std::clog << "Package file is up-to-date" << std::endl;
 443          // list cleanup needs to know that this file as well as the already
 444          // present index is ours, so we create an empty diff to save it for us
 445          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 446                ExpectedHash, ServerSha1, available_patches);
 447          return true;
 448       }
 449       else
 450       {
 451          if(Debug)
 452             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 453
 454          // check the historie and see what patches we need
 455          string const history = Tags.FindS("SHA1-History");
 456          std::stringstream hist(history);
 457          while(hist >> d.sha1 >> size >> d.file)
 458          {
 459             // read until the first match is found
 460             // from that point on, we probably need all diffs
 461             if(d.sha1 == local_sha1)
 462                found=true;
 463             else if (found == false)
 464                continue;
 465
 466             if(Debug)
 467                std::clog << "Need to get diff: " << d.file << std::endl;
 468             available_patches.push_back(d);
 469          }
 470
 471          if (available_patches.empty() == false)
 472          {
 473             // patching with too many files is rather slow compared to a fast download
 474             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 475             if (fileLimit != 0 && fileLimit < available_patches.size())
 476             {
 477                if (Debug)
 478                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 479                         << ") so fallback to complete download" << std::endl;
 480                return false;
 481             }
 482
 483             // see if the patches are too big
 484             found = false; // it was true and it will be true again at the end
 485             d = *available_patches.begin();
 486             string const firstPatch = d.file;
 487             unsigned long patchesSize = 0;
 488             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 489             while(patches >> d.sha1 >> size >> d.file)
 490             {
 491                if (firstPatch == d.file)
 492                   found = true;
 493                else if (found == false)
 494                   continue;
 495
 496                patchesSize += atol(size.c_str());
 497             }
 498             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 499             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 500             {
 501                if (Debug)
 502                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 503                         << ") so fallback to complete download" << std::endl;
 504                return false;
 505             }
 506          }
 507       }
 508
 509       // we have something, queue the next diff
 510       if(found)
 511       {
 512          // queue the diffs
 513          string::size_type const last_space = Description.rfind(" ");
 514          if(last_space != string::npos)
 515             Description.erase(last_space, Description.size()-last_space);
 516
 517          /* decide if we should download patches one by one or in one go:
 518             The first is good if the server merges patches, but many don't so client
 519             based merging can be attempt in which case the second is better.
 520             "bad things" will happen if patches are merged on the server,
 521             but client side merging is attempt as well */
 522          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 523          if (pdiff_merge == true)
 524          {
 525             // reprepro adds this flag if it has merged patches on the server
 526             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 527             pdiff_merge = (precedence != "merged");
 528          }
 529
 530          if (pdiff_merge == false)
 531             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 532                   ExpectedHash, ServerSha1, available_patches);
 533          else
 534          {
 535             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 536             for(size_t i = 0; i < available_patches.size(); ++i)
 537                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 538                      available_patches[i], diffs);
 539          }
 540
 541          Complete = false;
 542          Status = StatDone;
 543          Dequeue();
 544          return true;
 545       }
 546    }
 547
 548    // Nothing found, report and return false
 549    // Failing here is ok, if we return false later, the full
 550    // IndexFile is queued
 551    if(Debug)
 552       std::clog << "Can't find a patch in the index file" << std::endl;
 553    return false;
 554 }
 555                                                                         /*}}}*/
 556 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 557 {
 558    if(Debug)
 559       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 560                 << "Falling back to normal index file acquire" << std::endl;
 561
 562    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 563                    ExpectedHash);
 564
 565    Complete = false;
 566    Status = StatDone;
 567    Dequeue();
 568 }
 569                                                                         /*}}}*/
 570 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 571                            pkgAcquire::MethodConfig *Cnf)
 572 {
 573    if(Debug)
 574       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 575
 576    Item::Done(Message,Size,Md5Hash,Cnf);
 577
 578    string FinalFile;
 579    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 580
 581    // success in downloading the index
 582    // rename the index
 583    FinalFile += string(".IndexDiff");
 584    if(Debug)
 585       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 586                 << std::endl;
 587    Rename(DestFile,FinalFile);
 588    chmod(FinalFile.c_str(),0644);
 589    DestFile = FinalFile;
 590
 591    if(!ParseDiffIndex(DestFile))
 592       return Failed("", NULL);
 593
 594    Complete = true;
 595    Status = StatDone;
 596    Dequeue();
 597    return;
 598 }
 599                                                                         /*}}}*/
 600 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* The package diff is added to the queue. one object is constructed
 603  * for each diff and the index
 604  */
 605 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 606                                    string URI,string URIDesc,string ShortDesc,
 607                                    HashString ExpectedHash,
 608                                    string ServerSha1,
 609                                    vector<DiffInfo> diffs)
 610    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 611      available_patches(diffs), ServerSha1(ServerSha1)
 612 {
 613
 614    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 615    DestFile += URItoFileName(URI);
 616
 617    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 618
 619    Description = URIDesc;
 620    Desc.Owner = this;
 621    Desc.ShortDesc = ShortDesc;
 622
 623    if(available_patches.empty() == true)
 624    {
 625       // we are done (yeah!)
 626       Finish(true);
 627    }
 628    else
 629    {
 630       // get the next diff
 631       State = StateFetchDiff;
 632       QueueNextDiff();
 633    }
 634 }
 635                                                                         /*}}}*/
 636 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 637 {
 638    if(Debug)
 639       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 640                 << "Falling back to normal index file acquire" << std::endl;
 641    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 642                    ExpectedHash);
 643    Finish();
 644 }
 645                                                                         /*}}}*/
 646 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 647 void pkgAcqIndexDiffs::Finish(bool allDone)
 648 {
 649    // we restore the original name, this is required, otherwise
 650    // the file will be cleaned
 651    if(allDone)
 652    {
 653       DestFile = _config->FindDir("Dir::State::lists");
 654       DestFile += URItoFileName(RealURI);
 655
 656       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 657       {
 658          RenameOnError(HashSumMismatch);
 659          Dequeue();
 660          return;
 661       }
 662
 663       // this is for the "real" finish
 664       Complete = true;
 665       Status = StatDone;
 666       Dequeue();
 667       if(Debug)
 668          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 669       return;
 670    }
 671
 672    if(Debug)
 673       std::clog << "Finishing: " << Desc.URI << std::endl;
 674    Complete = false;
 675    Status = StatDone;
 676    Dequeue();
 677    return;
 678 }
 679                                                                         /*}}}*/
 680 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 681 {
 682
 683    // calc sha1 of the just patched file
 684    string FinalFile = _config->FindDir("Dir::State::lists");
 685    FinalFile += URItoFileName(RealURI);
 686
 687    FileFd fd(FinalFile, FileFd::ReadOnly);
 688    SHA1Summation SHA1;
 689    SHA1.AddFD(fd);
 690    string local_sha1 = string(SHA1.Result());
 691    if(Debug)
 692       std::clog << "QueueNextDiff: "
 693                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 694
 695    // final file reached before all patches are applied
 696    if(local_sha1 == ServerSha1)
 697    {
 698       Finish(true);
 699       return true;
 700    }
 701
 702    // remove all patches until the next matching patch is found
 703    // this requires the Index file to be ordered
 704    for(vector<DiffInfo>::iterator I=available_patches.begin();
 705        available_patches.empty() == false &&
 706           I != available_patches.end() &&
 707           I->sha1 != local_sha1;
 708        ++I)
 709    {
 710       available_patches.erase(I);
 711    }
 712
 713    // error checking and falling back if no patch was found
 714    if(available_patches.empty() == true)
 715    {
 716       Failed("", NULL);
 717       return false;
 718    }
 719
 720    // queue the right diff
 721    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 722    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 723    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 724    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 725
 726    if(Debug)
 727       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 728
 729    QueueURI(Desc);
 730
 731    return true;
 732 }
 733                                                                         /*}}}*/
 734 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 735                             pkgAcquire::MethodConfig *Cnf)
 736 {
 737    if(Debug)
 738       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 739
 740    Item::Done(Message,Size,Md5Hash,Cnf);
 741
 742    string FinalFile;
 743    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 744
 745    // success in downloading a diff, enter ApplyDiff state
 746    if(State == StateFetchDiff)
 747    {
 748
 749       // rred excepts the patch as $FinalFile.ed
 750       Rename(DestFile,FinalFile+".ed");
 751
 752       if(Debug)
 753          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 754
 755       State = StateApplyDiff;
 756       Local = true;
 757       Desc.URI = "rred:" + FinalFile;
 758       QueueURI(Desc);
 759       Mode = "rred";
 760       return;
 761    }
 762
 763
 764    // success in download/apply a diff, queue next (if needed)
 765    if(State == StateApplyDiff)
 766    {
 767       // remove the just applied patch
 768       available_patches.erase(available_patches.begin());
 769       unlink((FinalFile + ".ed").c_str());
 770
 771       // move into place
 772       if(Debug)
 773       {
 774          std::clog << "Moving patched file in place: " << std::endl
 775                    << DestFile << " -> " << FinalFile << std::endl;
 776       }
 777       Rename(DestFile,FinalFile);
 778       chmod(FinalFile.c_str(),0644);
 779
 780       // see if there is more to download
 781       if(available_patches.empty() == false) {
 782          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 783                               ExpectedHash, ServerSha1, available_patches);
 784          return Finish();
 785       } else
 786          return Finish(true);
 787    }
 788 }
 789                                                                         /*}}}*/
 790 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 791 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 792                                    string const &URI, string const &URIDesc,
 793                                    string const &ShortDesc, HashString const &ExpectedHash,
 794                                    DiffInfo const &patch,
 795                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 796    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 797      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 798 {
 799
 800    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 801    DestFile += URItoFileName(URI);
 802
 803    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 804
 805    Description = URIDesc;
 806    Desc.Owner = this;
 807    Desc.ShortDesc = ShortDesc;
 808
 809    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 810    Desc.Description = Description + " " + patch.file + string(".pdiff");
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 813
 814    if(Debug)
 815       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 816
 817    QueueURI(Desc);
 818 }
 819                                                                         /*}}}*/
 820 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 821 {
 822    if(Debug)
 823       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 824    Complete = false;
 825    Status = StatDone;
 826    Dequeue();
 827
 828    // check if we are the first to fail, otherwise we are done here
 829    State = StateDoneDiff;
 830    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 831          I != allPatches->end(); ++I)
 832       if ((*I)->State == StateErrorDiff)
 833          return;
 834
 835    // first failure means we should fallback
 836    State = StateErrorDiff;
 837    std::clog << "Falling back to normal index file acquire" << std::endl;
 838    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 839                    ExpectedHash);
 840 }
 841                                                                         /*}}}*/
 842 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 843                             pkgAcquire::MethodConfig *Cnf)
 844 {
 845    if(Debug)
 846       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 847
 848    Item::Done(Message,Size,Md5Hash,Cnf);
 849
 850    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 851
 852    if (State == StateFetchDiff)
 853    {
 854       // rred expects the patch as $FinalFile.ed.$patchname.gz
 855       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 856
 857       // check if this is the last completed diff
 858       State = StateDoneDiff;
 859       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 860             I != allPatches->end(); ++I)
 861          if ((*I)->State != StateDoneDiff)
 862          {
 863             if(Debug)
 864                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 865             return;
 866          }
 867
 868       // this is the last completed diff, so we are ready to apply now
 869       State = StateApplyDiff;
 870
 871       if(Debug)
 872          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 873
 874       Local = true;
 875       Desc.URI = "rred:" + FinalFile;
 876       QueueURI(Desc);
 877       Mode = "rred";
 878       return;
 879    }
 880    // success in download/apply all diffs, clean up
 881    else if (State == StateApplyDiff)
 882    {
 883       // see if we really got the expected file
 884       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 885       {
 886          RenameOnError(HashSumMismatch);
 887          return;
 888       }
 889
 890       // move the result into place
 891       if(Debug)
 892          std::clog << "Moving patched file in place: " << std::endl
 893                    << DestFile << " -> " << FinalFile << std::endl;
 894       Rename(DestFile, FinalFile);
 895       chmod(FinalFile.c_str(), 0644);
 896
 897       // otherwise lists cleanup will eat the file
 898       DestFile = FinalFile;
 899
 900       // ensure the ed's are gone regardless of list-cleanup
 901       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 902             I != allPatches->end(); ++I)
 903       {
 904             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 905             unlink(patch.c_str());
 906       }
 907
 908       // all set and done
 909       Complete = true;
 910       if(Debug)
 911          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 912    }
 913 }
 914                                                                         /*}}}*/
 915 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* The package file is added to the queue and a second class is
 918    instantiated to fetch the revision file */
 919 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 920                          string URI,string URIDesc,string ShortDesc,
 921                          HashString ExpectedHash, string comprExt)
 922    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 923 {
 924    if(comprExt.empty() == true)
 925    {
 926       // autoselect the compression method
 927       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 928       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 929          comprExt.append(*t).append(" ");
 930       if (comprExt.empty() == false)
 931          comprExt.erase(comprExt.end()-1);
 932    }
 933    CompressionExtension = comprExt;
 934
 935    Init(URI, URIDesc, ShortDesc);
 936 }
 937 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 938                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 939    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 940 {
 941    // autoselect the compression method
 942    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 943    CompressionExtension = "";
 944    if (ExpectedHash.empty() == false)
 945    {
 946       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 947          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 948             CompressionExtension.append(*t).append(" ");
 949    }
 950    else
 951    {
 952       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 953          CompressionExtension.append(*t).append(" ");
 954    }
 955    if (CompressionExtension.empty() == false)
 956       CompressionExtension.erase(CompressionExtension.end()-1);
 957
 958    Init(Target->URI, Target->Description, Target->ShortDesc);
 959 }
 960                                                                         /*}}}*/
 961 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 962 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 963    Decompression = false;
 964    Erase = false;
 965
 966    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 967    DestFile += URItoFileName(URI);
 968
 969    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 970    if (comprExt == "uncompressed")
 971       Desc.URI = URI;
 972    else {
 973       Desc.URI = URI + '.' + comprExt;
 974       DestFile = DestFile + '.' + comprExt;
 975    }
 976
 977    Desc.Description = URIDesc;
 978    Desc.Owner = this;
 979    Desc.ShortDesc = ShortDesc;
 980
 981    QueueURI(Desc);
 982 }
 983                                                                         /*}}}*/
 984 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 985 // ---------------------------------------------------------------------
 986 /* The only header we use is the last-modified header. */
 987 string pkgAcqIndex::Custom600Headers()
 988 {
 989    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 990    string Final = _config->FindDir("Dir::State::lists");
 991    Final += URItoFileName(RealURI);
 992    if (_config->FindB("Acquire::GzipIndexes",false))
 993       Final += compExt;
 994
 995    string msg = "\nIndex-File: true";
 996    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 997    //        seems to be difficult without breaking ABI
 998    if (ShortDesc().find("Translation") != 0)
 999       msg += "\nFail-Ignore: true";
1000    struct stat Buf;
1001    if (stat(Final.c_str(),&Buf) == 0)
1002       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1003
1004    return msg;
1005 }
1006                                                                         /*}}}*/
1007 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1008 {
1009    size_t const nextExt = CompressionExtension.find(' ');
1010    if (nextExt != std::string::npos)
1011    {
1012       CompressionExtension = CompressionExtension.substr(nextExt+1);
1013       Init(RealURI, Desc.Description, Desc.ShortDesc);
1014       return;
1015    }
1016
1017    // on decompression failure, remove bad versions in partial/
1018    if (Decompression && Erase) {
1019       string s = _config->FindDir("Dir::State::lists") + "partial/";
1020       s.append(URItoFileName(RealURI));
1021       unlink(s.c_str());
1022    }
1023
1024    Item::Failed(Message,Cnf);
1025 }
1026                                                                         /*}}}*/
1027 // pkgAcqIndex::GetFinalFilename - Return the full final file path      /*{{{*/
1028 std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
1029                                           std::string const &compExt)
1030 {
1031    std::string FinalFile = _config->FindDir("Dir::State::lists");
1032    FinalFile += URItoFileName(URI);
1033    if (_config->FindB("Acquire::GzipIndexes",false) == true)
1034       FinalFile += '.' + compExt;
1035    return FinalFile;
1036 }
1037                                                                         /*}}}*/
1038 // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit         /*{{{*/
1039 void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName)
1040 {
1041    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1042    if (_config->FindB("Acquire::GzipIndexes",false) == true)
1043       DestFile += compExt;
1044
1045    string FinalFile = GetFinalFilename(RealURI, compExt);
1046    Rename(FinalFile, FileName);
1047    Decompression = true;
1048    Desc.URI = "copy:" + FileName;
1049    QueueURI(Desc);
1050 }
1051                                                                         /*}}}*/
1052 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1053 // ---------------------------------------------------------------------
1054 /* This goes through a number of states.. On the initial fetch the
1055    method could possibly return an alternate filename which points
1056    to the uncompressed version of the file. If this is so the file
1057    is copied into the partial directory. In all other cases the file
1058    is decompressed with a gzip uri. */
1059 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1060                        pkgAcquire::MethodConfig *Cfg)
1061 {
1062    Item::Done(Message,Size,Hash,Cfg);
1063    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1064
1065    if (Decompression == true)
1066    {
1067       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1068       {
1069          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1070          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1071       }
1072
1073       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1074       {
1075          Desc.URI = RealURI;
1076          RenameOnError(HashSumMismatch);
1077          return;
1078       }
1079
1080       // FIXME: this can go away once we only ever download stuff that
1081       //        has a valid hash and we never do GET based probing
1082       //
1083       /* Always verify the index file for correctness (all indexes must
1084        * have a Package field) (LP: #346386) (Closes: #627642)
1085        */
1086       FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension);
1087       // Only test for correctness if the file is not empty (empty is ok)
1088       if (fd.Size() > 0)
1089       {
1090          pkgTagSection sec;
1091          pkgTagFile tag(&fd);
1092
1093          // all our current indexes have a field 'Package' in each section
1094          if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1095          {
1096             RenameOnError(InvalidFormat);
1097             return;
1098          }
1099       }
1100
1101       // Done, move it into position
1102       string FinalFile = GetFinalFilename(RealURI, compExt);
1103       Rename(DestFile,FinalFile);
1104       chmod(FinalFile.c_str(),0644);
1105
1106       /* We restore the original name to DestFile so that the clean operation
1107          will work OK */
1108       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1109       DestFile += URItoFileName(RealURI);
1110       if (_config->FindB("Acquire::GzipIndexes",false))
1111          DestFile += '.' + compExt;
1112
1113       // Remove the compressed version.
1114       if (Erase == true)
1115          unlink(DestFile.c_str());
1116       return;
1117    }
1118
1119    Erase = false;
1120    Complete = true;
1121
1122    // Handle the unzipd case
1123    string FileName = LookupTag(Message,"Alt-Filename");
1124    if (FileName.empty() == false)
1125    {
1126       Decompression = true;
1127       Local = true;
1128       DestFile += ".decomp";
1129       Desc.URI = "copy:" + FileName;
1130       QueueURI(Desc);
1131       Mode = "copy";
1132       return;
1133    }
1134
1135    FileName = LookupTag(Message,"Filename");
1136    if (FileName.empty() == true)
1137    {
1138       Status = StatError;
1139       ErrorText = "Method gave a blank filename";
1140    }
1141
1142    if (FileName == DestFile)
1143       Erase = true;
1144    else
1145       Local = true;
1146
1147    // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
1148    // file when its doing the indexcopy
1149    if (RealURI.substr(0,6) == "cdrom:" &&
1150        StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1151       return;
1152
1153    // The files timestamp matches, for non-local URLs reverify the local
1154    // file, for local file, uncompress again to ensure the hashsum is still
1155    // matching the Release file
1156    if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1157    {
1158       // set destfile to the final destfile
1159       if(_config->FindB("Acquire::GzipIndexes",false) == false)
1160       {
1161          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1162          DestFile += URItoFileName(RealURI);
1163       }
1164
1165       ReverifyAfterIMS(FileName);
1166       return;
1167    }
1168    string decompProg;
1169
1170    // If we enable compressed indexes, queue for hash verification
1171    if (_config->FindB("Acquire::GzipIndexes",false))
1172    {
1173       DestFile = _config->FindDir("Dir::State::lists");
1174       DestFile += URItoFileName(RealURI) + '.' + compExt;
1175
1176       Decompression = true;
1177       Desc.URI = "copy:" + FileName;
1178       QueueURI(Desc);
1179
1180       return;
1181     }
1182
1183    // get the binary name for your used compression type
1184    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1185    if(decompProg.empty() == false);
1186    else if(compExt == "uncompressed")
1187       decompProg = "copy";
1188    else {
1189       _error->Error("Unsupported extension: %s", compExt.c_str());
1190       return;
1191    }
1192
1193    Decompression = true;
1194    DestFile += ".decomp";
1195    Desc.URI = decompProg + ":" + FileName;
1196    QueueURI(Desc);
1197
1198    // FIXME: this points to a c++ string that goes out of scope
1199    Mode = decompProg.c_str();
1200 }
1201                                                                         /*}}}*/
1202 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1203 // ---------------------------------------------------------------------
1204 /* The Translation file is added to the queue */
1205 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1206                             string URI,string URIDesc,string ShortDesc)
1207   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1208 {
1209 }
1210 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1211                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1212   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1213 {
1214 }
1215                                                                         /*}}}*/
1216 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1217 // ---------------------------------------------------------------------
1218 string pkgAcqIndexTrans::Custom600Headers()
1219 {
1220    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1221    string Final = _config->FindDir("Dir::State::lists");
1222    Final += URItoFileName(RealURI);
1223    if (_config->FindB("Acquire::GzipIndexes",false))
1224       Final += compExt;
1225
1226    struct stat Buf;
1227    if (stat(Final.c_str(),&Buf) != 0)
1228       return "\nFail-Ignore: true\nIndex-File: true";
1229    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1230 }
1231                                                                         /*}}}*/
1232 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1233 // ---------------------------------------------------------------------
1234 /* */
1235 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1236 {
1237    size_t const nextExt = CompressionExtension.find(' ');
1238    if (nextExt != std::string::npos)
1239    {
1240       CompressionExtension = CompressionExtension.substr(nextExt+1);
1241       Init(RealURI, Desc.Description, Desc.ShortDesc);
1242       Status = StatIdle;
1243       return;
1244    }
1245
1246    if (Cnf->LocalOnly == true ||
1247        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1248    {
1249       // Ignore this
1250       Status = StatDone;
1251       Complete = false;
1252       Dequeue();
1253       return;
1254    }
1255
1256    Item::Failed(Message,Cnf);
1257 }
1258                                                                         /*}}}*/
1259 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1260                              string URI,string URIDesc,string ShortDesc,
1261                              string MetaIndexURI, string MetaIndexURIDesc,
1262                              string MetaIndexShortDesc,
1263                              const vector<IndexTarget*>* IndexTargets,
1264                              indexRecords* MetaIndexParser) :
1265    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1266    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1267    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1268 {
1269    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1270    DestFile += URItoFileName(URI);
1271
1272    // remove any partial downloaded sig-file in partial/.
1273    // it may confuse proxies and is too small to warrant a
1274    // partial download anyway
1275    unlink(DestFile.c_str());
1276
1277    // Create the item
1278    Desc.Description = URIDesc;
1279    Desc.Owner = this;
1280    Desc.ShortDesc = ShortDesc;
1281    Desc.URI = URI;
1282
1283    string Final = _config->FindDir("Dir::State::lists");
1284    Final += URItoFileName(RealURI);
1285    if (RealFileExists(Final) == true)
1286    {
1287       // File was already in place.  It needs to be re-downloaded/verified
1288       // because Release might have changed, we do give it a different
1289       // name than DestFile because otherwise the http method will
1290       // send If-Range requests and there are too many broken servers
1291       // out there that do not understand them
1292       LastGoodSig = DestFile+".reverify";
1293       Rename(Final,LastGoodSig);
1294    }
1295
1296    QueueURI(Desc);
1297 }
1298                                                                         /*}}}*/
1299 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1300 {
1301    // if the file was never queued undo file-changes done in the constructor
1302    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1303          LastGoodSig.empty() == false)
1304    {
1305       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1306       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1307          Rename(LastGoodSig, Final);
1308    }
1309
1310 }
1311                                                                         /*}}}*/
1312 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1313 // ---------------------------------------------------------------------
1314 /* The only header we use is the last-modified header. */
1315 string pkgAcqMetaSig::Custom600Headers()
1316 {
1317    struct stat Buf;
1318    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1319       return "\nIndex-File: true";
1320
1321    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1322 }
1323
1324 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1325                          pkgAcquire::MethodConfig *Cfg)
1326 {
1327    Item::Done(Message,Size,MD5,Cfg);
1328
1329    string FileName = LookupTag(Message,"Filename");
1330    if (FileName.empty() == true)
1331    {
1332       Status = StatError;
1333       ErrorText = "Method gave a blank filename";
1334       return;
1335    }
1336
1337    if (FileName != DestFile)
1338    {
1339       // We have to copy it into place
1340       Local = true;
1341       Desc.URI = "copy:" + FileName;
1342       QueueURI(Desc);
1343       return;
1344    }
1345
1346    Complete = true;
1347
1348    // put the last known good file back on i-m-s hit (it will
1349    // be re-verified again)
1350    // Else do nothing, we have the new file in DestFile then
1351    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1352       Rename(LastGoodSig, DestFile);
1353
1354    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1355    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1356                        MetaIndexShortDesc,  DestFile, IndexTargets,
1357                        MetaIndexParser);
1358
1359 }
1360                                                                         /*}}}*/
1361 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1362 {
1363    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1364
1365    // if we get a network error we fail gracefully
1366    if(Status == StatTransientNetworkError)
1367    {
1368       Item::Failed(Message,Cnf);
1369       // move the sigfile back on transient network failures
1370       if(FileExists(LastGoodSig))
1371          Rename(LastGoodSig,Final);
1372
1373       // set the status back to , Item::Failed likes to reset it
1374       Status = pkgAcquire::Item::StatTransientNetworkError;
1375       return;
1376    }
1377
1378    // Delete any existing sigfile when the acquire failed
1379    unlink(Final.c_str());
1380
1381    // queue a pkgAcqMetaIndex with no sigfile
1382    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1383                        "", IndexTargets, MetaIndexParser);
1384
1385    if (Cnf->LocalOnly == true ||
1386        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1387    {
1388       // Ignore this
1389       Status = StatDone;
1390       Complete = false;
1391       Dequeue();
1392       return;
1393    }
1394
1395    Item::Failed(Message,Cnf);
1396 }
1397                                                                         /*}}}*/
1398 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1399                                  string URI,string URIDesc,string ShortDesc,
1400                                  string SigFile,
1401                                  const vector<struct IndexTarget*>* IndexTargets,
1402                                  indexRecords* MetaIndexParser) :
1403    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1404    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1405 {
1406    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1407    DestFile += URItoFileName(URI);
1408
1409    // Create the item
1410    Desc.Description = URIDesc;
1411    Desc.Owner = this;
1412    Desc.ShortDesc = ShortDesc;
1413    Desc.URI = URI;
1414
1415    QueueURI(Desc);
1416 }
1417                                                                         /*}}}*/
1418 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1419 // ---------------------------------------------------------------------
1420 /* The only header we use is the last-modified header. */
1421 string pkgAcqMetaIndex::Custom600Headers()
1422 {
1423    string Final = _config->FindDir("Dir::State::lists");
1424    Final += URItoFileName(RealURI);
1425
1426    struct stat Buf;
1427    if (stat(Final.c_str(),&Buf) != 0)
1428       return "\nIndex-File: true";
1429
1430    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1431 }
1432                                                                         /*}}}*/
1433 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1434                            pkgAcquire::MethodConfig *Cfg)
1435 {
1436    Item::Done(Message,Size,Hash,Cfg);
1437
1438    // MetaIndexes are done in two passes: one to download the
1439    // metaindex with an appropriate method, and a second to verify it
1440    // with the gpgv method
1441
1442    if (AuthPass == true)
1443    {
1444       AuthDone(Message);
1445
1446       // all cool, move Release file into place
1447       Complete = true;
1448    }
1449    else
1450    {
1451       RetrievalDone(Message);
1452       if (!Complete)
1453          // Still more retrieving to do
1454          return;
1455
1456       if (SigFile == "")
1457       {
1458          // There was no signature file, so we are finished.  Download
1459          // the indexes and do only hashsum verification if possible
1460          MetaIndexParser->Load(DestFile);
1461          QueueIndexes(false);
1462       }
1463       else
1464       {
1465          // There was a signature file, so pass it to gpgv for
1466          // verification
1467
1468          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1469             std::cerr << "Metaindex acquired, queueing gpg verification ("
1470                       << SigFile << "," << DestFile << ")\n";
1471          AuthPass = true;
1472          Desc.URI = "gpgv:" + SigFile;
1473          QueueURI(Desc);
1474          Mode = "gpgv";
1475          return;
1476       }
1477    }
1478
1479    if (Complete == true)
1480    {
1481       string FinalFile = _config->FindDir("Dir::State::lists");
1482       FinalFile += URItoFileName(RealURI);
1483       if (SigFile == DestFile)
1484          SigFile = FinalFile;
1485       Rename(DestFile,FinalFile);
1486       chmod(FinalFile.c_str(),0644);
1487       DestFile = FinalFile;
1488    }
1489 }
1490                                                                         /*}}}*/
1491 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1492 {
1493    // We have just finished downloading a Release file (it is not
1494    // verified yet)
1495
1496    string FileName = LookupTag(Message,"Filename");
1497    if (FileName.empty() == true)
1498    {
1499       Status = StatError;
1500       ErrorText = "Method gave a blank filename";
1501       return;
1502    }
1503
1504    if (FileName != DestFile)
1505    {
1506       Local = true;
1507       Desc.URI = "copy:" + FileName;
1508       QueueURI(Desc);
1509       return;
1510    }
1511
1512    // make sure to verify against the right file on I-M-S hit
1513    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1514    if(IMSHit)
1515    {
1516       string FinalFile = _config->FindDir("Dir::State::lists");
1517       FinalFile += URItoFileName(RealURI);
1518       if (SigFile == DestFile)
1519       {
1520          SigFile = FinalFile;
1521          // constructor of pkgAcqMetaClearSig moved it out of the way,
1522          // now move it back in on IMS hit for the 'old' file
1523          string const OldClearSig = DestFile + ".reverify";
1524          if (RealFileExists(OldClearSig) == true)
1525             Rename(OldClearSig, FinalFile);
1526       }
1527       DestFile = FinalFile;
1528    }
1529    Complete = true;
1530 }
1531                                                                         /*}}}*/
1532 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1533 {
1534    // At this point, the gpgv method has succeeded, so there is a
1535    // valid signature from a key in the trusted keyring.  We
1536    // perform additional verification of its contents, and use them
1537    // to verify the indexes we are about to download
1538
1539    if (!MetaIndexParser->Load(DestFile))
1540    {
1541       Status = StatAuthError;
1542       ErrorText = MetaIndexParser->ErrorText;
1543       return;
1544    }
1545
1546    if (!VerifyVendor(Message))
1547    {
1548       return;
1549    }
1550
1551    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1552       std::cerr << "Signature verification succeeded: "
1553                 << DestFile << std::endl;
1554
1555    // do not trust any previously unverified content that we may have
1556    string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1557    if (DestFile != SigFile)
1558       LastGoodSigFile.append(".gpg");
1559    LastGoodSigFile.append(".reverify");
1560    if(IMSHit == false && RealFileExists(LastGoodSigFile) == false)
1561    {
1562       for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1563            Target != IndexTargets->end();
1564            ++Target)
1565       {
1566          // remove old indexes
1567          std::string index = _config->FindDir("Dir::State::lists") +
1568             URItoFileName((*Target)->URI);
1569          unlink(index.c_str());
1570          // and also old gzipindexes
1571          std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1572          for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1573          {
1574             index += '.' + (*t);
1575             unlink(index.c_str());
1576          }
1577       }
1578    }
1579
1580
1581    // Download further indexes with verification
1582    QueueIndexes(true);
1583
1584    // is it a clearsigned MetaIndex file?
1585    if (DestFile == SigFile)
1586       return;
1587
1588    // Done, move signature file into position
1589    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1590       URItoFileName(RealURI) + ".gpg";
1591    Rename(SigFile,VerifiedSigFile);
1592    chmod(VerifiedSigFile.c_str(),0644);
1593 }
1594                                                                         /*}}}*/
1595 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1596 {
1597 #if 0
1598    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1599     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1600    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1601    {
1602       Status = StatError;
1603       ErrorText = MetaIndexParser->ErrorText;
1604       return;
1605    }
1606 #endif
1607    bool transInRelease = false;
1608    {
1609       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1610       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1611          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1612          if (k->find("Translation-") != std::string::npos)
1613          {
1614             transInRelease = true;
1615             break;
1616          }
1617    }
1618
1619    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1620         Target != IndexTargets->end();
1621         ++Target)
1622    {
1623       HashString ExpectedIndexHash;
1624       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1625       bool compressedAvailable = false;
1626       if (Record == NULL)
1627       {
1628          if ((*Target)->IsOptional() == true)
1629          {
1630             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1631             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1632                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1633                {
1634                   compressedAvailable = true;
1635                   break;
1636                }
1637          }
1638          else if (verify == true)
1639          {
1640             Status = StatAuthError;
1641             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1642             return;
1643          }
1644       }
1645       else
1646       {
1647          ExpectedIndexHash = Record->Hash;
1648          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1649          {
1650             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1651             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1652             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1653          }
1654          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1655          {
1656             Status = StatAuthError;
1657             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1658             return;
1659          }
1660       }
1661
1662       if ((*Target)->IsOptional() == true)
1663       {
1664          if ((*Target)->IsSubIndex() == true)
1665             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1666                                 (*Target)->ShortDesc, ExpectedIndexHash);
1667          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1668          {
1669             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1670                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1671                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1672                                    (*Target)->ShortDesc, ExpectedIndexHash);
1673             else
1674                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1675          }
1676          continue;
1677       }
1678
1679       /* Queue Packages file (either diff or full packages files, depending
1680          on the users option) - we also check if the PDiff Index file is listed
1681          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1682          instead, but passing the required info to it is to much hassle */
1683       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1684           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1685          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1686                              (*Target)->ShortDesc, ExpectedIndexHash);
1687       else
1688          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1689    }
1690 }
1691                                                                         /*}}}*/
1692 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1693 {
1694    string::size_type pos;
1695
1696    // check for missing sigs (that where not fatal because otherwise we had
1697    // bombed earlier)
1698    string missingkeys;
1699    string msg = _("There is no public key available for the "
1700                   "following key IDs:\n");
1701    pos = Message.find("NO_PUBKEY ");
1702    if (pos != std::string::npos)
1703    {
1704       string::size_type start = pos+strlen("NO_PUBKEY ");
1705       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1706       missingkeys += (Fingerprint);
1707    }
1708    if(!missingkeys.empty())
1709       _error->Warning("%s", (msg + missingkeys).c_str());
1710
1711    string Transformed = MetaIndexParser->GetExpectedDist();
1712
1713    if (Transformed == "../project/experimental")
1714    {
1715       Transformed = "experimental";
1716    }
1717
1718    pos = Transformed.rfind('/');
1719    if (pos != string::npos)
1720    {
1721       Transformed = Transformed.substr(0, pos);
1722    }
1723
1724    if (Transformed == ".")
1725    {
1726       Transformed = "";
1727    }
1728
1729    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1730        MetaIndexParser->GetValidUntil() > 0) {
1731       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1732       if (invalid_since > 0)
1733          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1734          // the time since then the file is invalid - formated in the same way as in
1735          // the download progress display (e.g. 7d 3h 42min 1s)
1736          return _error->Error(
1737             _("Release file for %s is expired (invalid since %s). "
1738               "Updates for this repository will not be applied."),
1739             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1740    }
1741
1742    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1743    {
1744       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1745       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1746       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1747    }
1748
1749    if (MetaIndexParser->CheckDist(Transformed) == false)
1750    {
1751       // This might become fatal one day
1752 //       Status = StatAuthError;
1753 //       ErrorText = "Conflicting distribution; expected "
1754 //          + MetaIndexParser->GetExpectedDist() + " but got "
1755 //          + MetaIndexParser->GetDist();
1756 //       return false;
1757       if (!Transformed.empty())
1758       {
1759          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1760                          Desc.Description.c_str(),
1761                          Transformed.c_str(),
1762                          MetaIndexParser->GetDist().c_str());
1763       }
1764    }
1765
1766    return true;
1767 }
1768                                                                         /*}}}*/
1769 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1770 // ---------------------------------------------------------------------
1771 /* */
1772 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1773 {
1774    if (AuthPass == true)
1775    {
1776       // gpgv method failed, if we have a good signature
1777       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1778       if (DestFile != SigFile)
1779          LastGoodSigFile.append(".gpg");
1780       LastGoodSigFile.append(".reverify");
1781
1782       if(FileExists(LastGoodSigFile))
1783       {
1784          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1785          if (DestFile != SigFile)
1786             VerifiedSigFile.append(".gpg");
1787          Rename(LastGoodSigFile, VerifiedSigFile);
1788          Status = StatTransientNetworkError;
1789          _error->Warning(_("An error occurred during the signature "
1790                            "verification. The repository is not updated "
1791                            "and the previous index files will be used. "
1792                            "GPG error: %s: %s\n"),
1793                          Desc.Description.c_str(),
1794                          LookupTag(Message,"Message").c_str());
1795          RunScripts("APT::Update::Auth-Failure");
1796          return;
1797       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1798          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1799          _error->Error(_("GPG error: %s: %s"),
1800                          Desc.Description.c_str(),
1801                          LookupTag(Message,"Message").c_str());
1802          return;
1803       } else {
1804          _error->Warning(_("GPG error: %s: %s"),
1805                          Desc.Description.c_str(),
1806                          LookupTag(Message,"Message").c_str());
1807       }
1808       // gpgv method failed
1809       ReportMirrorFailure("GPGFailure");
1810    }
1811
1812    /* Always move the meta index, even if gpgv failed. This ensures
1813     * that PackageFile objects are correctly filled in */
1814    if (FileExists(DestFile)) {
1815       string FinalFile = _config->FindDir("Dir::State::lists");
1816       FinalFile += URItoFileName(RealURI);
1817       /* InRelease files become Release files, otherwise
1818        * they would be considered as trusted later on */
1819       if (SigFile == DestFile) {
1820          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1821                                        "Release");
1822          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1823                                        "Release");
1824          SigFile = FinalFile;
1825       }
1826       Rename(DestFile,FinalFile);
1827       chmod(FinalFile.c_str(),0644);
1828
1829       DestFile = FinalFile;
1830    }
1831
1832    // No Release file was present, or verification failed, so fall
1833    // back to queueing Packages files without verification
1834    QueueIndexes(false);
1835 }
1836                                                                         /*}}}*/
1837 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1838                 string const &URI, string const &URIDesc, string const &ShortDesc,
1839                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1840                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1841                 const vector<struct IndexTarget*>* IndexTargets,
1842                 indexRecords* MetaIndexParser) :
1843         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1844         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1845         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1846 {
1847    SigFile = DestFile;
1848
1849    // keep the old InRelease around in case of transistent network errors
1850    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1851    if (RealFileExists(Final) == true)
1852    {
1853       string const LastGoodSig = DestFile + ".reverify";
1854       Rename(Final,LastGoodSig);
1855    }
1856 }
1857                                                                         /*}}}*/
1858 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1859 {
1860    // if the file was never queued undo file-changes done in the constructor
1861    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1862    {
1863       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1864       string const LastGoodSig = DestFile + ".reverify";
1865       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1866          Rename(LastGoodSig, Final);
1867    }
1868 }
1869                                                                         /*}}}*/
1870 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1871 // ---------------------------------------------------------------------
1872 // FIXME: this can go away once the InRelease file is used widely
1873 string pkgAcqMetaClearSig::Custom600Headers()
1874 {
1875    string Final = _config->FindDir("Dir::State::lists");
1876    Final += URItoFileName(RealURI);
1877
1878    struct stat Buf;
1879    if (stat(Final.c_str(),&Buf) != 0)
1880    {
1881       Final = DestFile + ".reverify";
1882       if (stat(Final.c_str(),&Buf) != 0)
1883          return "\nIndex-File: true\nFail-Ignore: true\n";
1884    }
1885
1886    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1887 }
1888                                                                         /*}}}*/
1889 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1890 {
1891    if (AuthPass == false)
1892    {
1893       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1894       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1895       string FinalFile = _config->FindDir("Dir::State::lists");
1896       FinalFile.append(URItoFileName(RealURI));
1897       if (FileExists(FinalFile))
1898          unlink(FinalFile.c_str());
1899
1900       new pkgAcqMetaSig(Owner,
1901                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1902                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1903                         IndexTargets, MetaIndexParser);
1904       if (Cnf->LocalOnly == true ||
1905           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1906          Dequeue();
1907    }
1908    else
1909       pkgAcqMetaIndex::Failed(Message, Cnf);
1910 }
1911                                                                         /*}}}*/
1912 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1913 // ---------------------------------------------------------------------
1914 /* This just sets up the initial fetch environment and queues the first
1915    possibilitiy */
1916 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1917                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1918                              string &StoreFilename) :
1919                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1920                StoreFilename(StoreFilename), Vf(Version.FileList()),
1921                Trusted(false)
1922 {
1923    Retries = _config->FindI("Acquire::Retries",0);
1924
1925    if (Version.Arch() == 0)
1926    {
1927       _error->Error(_("I wasn't able to locate a file for the %s package. "
1928                       "This might mean you need to manually fix this package. "
1929                       "(due to missing arch)"),
1930                     Version.ParentPkg().FullName().c_str());
1931       return;
1932    }
1933
1934    /* We need to find a filename to determine the extension. We make the
1935       assumption here that all the available sources for this version share
1936       the same extension.. */
1937    // Skip not source sources, they do not have file fields.
1938    for (; Vf.end() == false; ++Vf)
1939    {
1940       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1941          continue;
1942       break;
1943    }
1944
1945    // Does not really matter here.. we are going to fail out below
1946    if (Vf.end() != true)
1947    {
1948       // If this fails to get a file name we will bomb out below.
1949       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1950       if (_error->PendingError() == true)
1951          return;
1952
1953       // Generate the final file name as: package_version_arch.foo
1954       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1955                       QuoteString(Version.VerStr(),"_:") + '_' +
1956                       QuoteString(Version.Arch(),"_:.") +
1957                       "." + flExtension(Parse.FileName());
1958    }
1959
1960    // check if we have one trusted source for the package. if so, switch
1961    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1962    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1963    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1964    bool seenUntrusted = false;
1965    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1966    {
1967       pkgIndexFile *Index;
1968       if (Sources->FindIndex(i.File(),Index) == false)
1969          continue;
1970
1971       if (debugAuth == true)
1972          std::cerr << "Checking index: " << Index->Describe()
1973                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1974
1975       if (Index->IsTrusted() == true)
1976       {
1977          Trusted = true;
1978          if (allowUnauth == false)
1979             break;
1980       }
1981       else
1982          seenUntrusted = true;
1983    }
1984
1985    // "allow-unauthenticated" restores apts old fetching behaviour
1986    // that means that e.g. unauthenticated file:// uris are higher
1987    // priority than authenticated http:// uris
1988    if (allowUnauth == true && seenUntrusted == true)
1989       Trusted = false;
1990
1991    // Select a source
1992    if (QueueNext() == false && _error->PendingError() == false)
1993       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1994                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1995 }
1996                                                                         /*}}}*/
1997 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1998 // ---------------------------------------------------------------------
1999 /* This queues the next available file version for download. It checks if
2000    the archive is already available in the cache and stashs the MD5 for
2001    checking later. */
2002 bool pkgAcqArchive::QueueNext()
2003 {
2004    string const ForceHash = _config->Find("Acquire::ForceHash");
2005    for (; Vf.end() == false; ++Vf)
2006    {
2007       // Ignore not source sources
2008       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2009          continue;
2010
2011       // Try to cross match against the source list
2012       pkgIndexFile *Index;
2013       if (Sources->FindIndex(Vf.File(),Index) == false)
2014             continue;
2015
2016       // only try to get a trusted package from another source if that source
2017       // is also trusted
2018       if(Trusted && !Index->IsTrusted())
2019          continue;
2020
2021       // Grab the text package record
2022       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2023       if (_error->PendingError() == true)
2024          return false;
2025
2026       string PkgFile = Parse.FileName();
2027       if (ForceHash.empty() == false)
2028       {
2029          if(stringcasecmp(ForceHash, "sha512") == 0)
2030             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
2031          else if(stringcasecmp(ForceHash, "sha256") == 0)
2032             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
2033          else if (stringcasecmp(ForceHash, "sha1") == 0)
2034             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
2035          else
2036             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2037       }
2038       else
2039       {
2040          string Hash;
2041          if ((Hash = Parse.SHA512Hash()).empty() == false)
2042             ExpectedHash = HashString("SHA512", Hash);
2043          else if ((Hash = Parse.SHA256Hash()).empty() == false)
2044             ExpectedHash = HashString("SHA256", Hash);
2045          else if ((Hash = Parse.SHA1Hash()).empty() == false)
2046             ExpectedHash = HashString("SHA1", Hash);
2047          else
2048             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2049       }
2050       if (PkgFile.empty() == true)
2051          return _error->Error(_("The package index files are corrupted. No Filename: "
2052                               "field for package %s."),
2053                               Version.ParentPkg().Name());
2054
2055       Desc.URI = Index->ArchiveURI(PkgFile);
2056       Desc.Description = Index->ArchiveInfo(Version);
2057       Desc.Owner = this;
2058       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2059
2060       // See if we already have the file. (Legacy filenames)
2061       FileSize = Version->Size;
2062       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2063       struct stat Buf;
2064       if (stat(FinalFile.c_str(),&Buf) == 0)
2065       {
2066          // Make sure the size matches
2067          if ((unsigned long long)Buf.st_size == Version->Size)
2068          {
2069             Complete = true;
2070             Local = true;
2071             Status = StatDone;
2072             StoreFilename = DestFile = FinalFile;
2073             return true;
2074          }
2075
2076          /* Hmm, we have a file and its size does not match, this means it is
2077             an old style mismatched arch */
2078          unlink(FinalFile.c_str());
2079       }
2080
2081       // Check it again using the new style output filenames
2082       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2083       if (stat(FinalFile.c_str(),&Buf) == 0)
2084       {
2085          // Make sure the size matches
2086          if ((unsigned long long)Buf.st_size == Version->Size)
2087          {
2088             Complete = true;
2089             Local = true;
2090             Status = StatDone;
2091             StoreFilename = DestFile = FinalFile;
2092             return true;
2093          }
2094
2095          /* Hmm, we have a file and its size does not match, this shouldn't
2096             happen.. */
2097          unlink(FinalFile.c_str());
2098       }
2099
2100       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2101
2102       // Check the destination file
2103       if (stat(DestFile.c_str(),&Buf) == 0)
2104       {
2105          // Hmm, the partial file is too big, erase it
2106          if ((unsigned long long)Buf.st_size > Version->Size)
2107             unlink(DestFile.c_str());
2108          else
2109             PartialSize = Buf.st_size;
2110       }
2111
2112       // Disables download of archives - useful if no real installation follows,
2113       // e.g. if we are just interested in proposed installation order
2114       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2115       {
2116          Complete = true;
2117          Local = true;
2118          Status = StatDone;
2119          StoreFilename = DestFile = FinalFile;
2120          return true;
2121       }
2122
2123       // Create the item
2124       Local = false;
2125       QueueURI(Desc);
2126
2127       ++Vf;
2128       return true;
2129    }
2130    return false;
2131 }
2132                                                                         /*}}}*/
2133 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2134 // ---------------------------------------------------------------------
2135 /* */
2136 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2137                          pkgAcquire::MethodConfig *Cfg)
2138 {
2139    Item::Done(Message,Size,CalcHash,Cfg);
2140
2141    // Check the size
2142    if (Size != Version->Size)
2143    {
2144       RenameOnError(SizeMismatch);
2145       return;
2146    }
2147
2148    // Check the hash
2149    if(ExpectedHash.toStr() != CalcHash)
2150    {
2151       RenameOnError(HashSumMismatch);
2152       return;
2153    }
2154
2155    // Grab the output filename
2156    string FileName = LookupTag(Message,"Filename");
2157    if (FileName.empty() == true)
2158    {
2159       Status = StatError;
2160       ErrorText = "Method gave a blank filename";
2161       return;
2162    }
2163
2164    Complete = true;
2165
2166    // Reference filename
2167    if (FileName != DestFile)
2168    {
2169       StoreFilename = DestFile = FileName;
2170       Local = true;
2171       return;
2172    }
2173
2174    // Done, move it into position
2175    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2176    FinalFile += flNotDir(StoreFilename);
2177    Rename(DestFile,FinalFile);
2178
2179    StoreFilename = DestFile = FinalFile;
2180    Complete = true;
2181 }
2182                                                                         /*}}}*/
2183 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2184 // ---------------------------------------------------------------------
2185 /* Here we try other sources */
2186 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2187 {
2188    ErrorText = LookupTag(Message,"Message");
2189
2190    /* We don't really want to retry on failed media swaps, this prevents
2191       that. An interesting observation is that permanent failures are not
2192       recorded. */
2193    if (Cnf->Removable == true &&
2194        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2195    {
2196       // Vf = Version.FileList();
2197       while (Vf.end() == false) ++Vf;
2198       StoreFilename = string();
2199       Item::Failed(Message,Cnf);
2200       return;
2201    }
2202
2203    if (QueueNext() == false)
2204    {
2205       // This is the retry counter
2206       if (Retries != 0 &&
2207           Cnf->LocalOnly == false &&
2208           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2209       {
2210          Retries--;
2211          Vf = Version.FileList();
2212          if (QueueNext() == true)
2213             return;
2214       }
2215
2216       StoreFilename = string();
2217       Item::Failed(Message,Cnf);
2218    }
2219 }
2220                                                                         /*}}}*/
2221 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2222 // ---------------------------------------------------------------------
2223 APT_PURE bool pkgAcqArchive::IsTrusted()
2224 {
2225    return Trusted;
2226 }
2227                                                                         /*}}}*/
2228 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2229 // ---------------------------------------------------------------------
2230 /* */
2231 void pkgAcqArchive::Finished()
2232 {
2233    if (Status == pkgAcquire::Item::StatDone &&
2234        Complete == true)
2235       return;
2236    StoreFilename = string();
2237 }
2238                                                                         /*}}}*/
2239 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2240 // ---------------------------------------------------------------------
2241 /* The file is added to the queue */
2242 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2243                        unsigned long long Size,string Dsc,string ShortDesc,
2244                        const string &DestDir, const string &DestFilename,
2245                        bool IsIndexFile) :
2246                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2247 {
2248    Retries = _config->FindI("Acquire::Retries",0);
2249
2250    if(!DestFilename.empty())
2251       DestFile = DestFilename;
2252    else if(!DestDir.empty())
2253       DestFile = DestDir + "/" + flNotDir(URI);
2254    else
2255       DestFile = flNotDir(URI);
2256
2257    // Create the item
2258    Desc.URI = URI;
2259    Desc.Description = Dsc;
2260    Desc.Owner = this;
2261
2262    // Set the short description to the archive component
2263    Desc.ShortDesc = ShortDesc;
2264
2265    // Get the transfer sizes
2266    FileSize = Size;
2267    struct stat Buf;
2268    if (stat(DestFile.c_str(),&Buf) == 0)
2269    {
2270       // Hmm, the partial file is too big, erase it
2271       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2272          unlink(DestFile.c_str());
2273       else
2274          PartialSize = Buf.st_size;
2275    }
2276
2277    QueueURI(Desc);
2278 }
2279                                                                         /*}}}*/
2280 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2281 // ---------------------------------------------------------------------
2282 /* */
2283 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2284                       pkgAcquire::MethodConfig *Cnf)
2285 {
2286    Item::Done(Message,Size,CalcHash,Cnf);
2287
2288    // Check the hash
2289    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2290    {
2291       RenameOnError(HashSumMismatch);
2292       return;
2293    }
2294
2295    string FileName = LookupTag(Message,"Filename");
2296    if (FileName.empty() == true)
2297    {
2298       Status = StatError;
2299       ErrorText = "Method gave a blank filename";
2300       return;
2301    }
2302
2303    Complete = true;
2304
2305    // The files timestamp matches
2306    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2307       return;
2308
2309    // We have to copy it into place
2310    if (FileName != DestFile)
2311    {
2312       Local = true;
2313       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2314           Cnf->Removable == true)
2315       {
2316          Desc.URI = "copy:" + FileName;
2317          QueueURI(Desc);
2318          return;
2319       }
2320
2321       // Erase the file if it is a symlink so we can overwrite it
2322       struct stat St;
2323       if (lstat(DestFile.c_str(),&St) == 0)
2324       {
2325          if (S_ISLNK(St.st_mode) != 0)
2326             unlink(DestFile.c_str());
2327       }
2328
2329       // Symlink the file
2330       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2331       {
2332          ErrorText = "Link to " + DestFile + " failure ";
2333          Status = StatError;
2334          Complete = false;
2335       }
2336    }
2337 }
2338                                                                         /*}}}*/
2339 // AcqFile::Failed - Failure handler                                    /*{{{*/
2340 // ---------------------------------------------------------------------
2341 /* Here we try other sources */
2342 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2343 {
2344    ErrorText = LookupTag(Message,"Message");
2345
2346    // This is the retry counter
2347    if (Retries != 0 &&
2348        Cnf->LocalOnly == false &&
2349        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2350    {
2351       Retries--;
2352       QueueURI(Desc);
2353       return;
2354    }
2355
2356    Item::Failed(Message,Cnf);
2357 }
2358                                                                         /*}}}*/
2359 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2360 // ---------------------------------------------------------------------
2361 /* The only header we use is the last-modified header. */
2362 string pkgAcqFile::Custom600Headers()
2363 {
2364    if (IsIndexFile)
2365       return "\nIndex-File: true";
2366    return "";
2367 }
2368                                                                         /*}}}*/